2024-11-17 09:40:26,513 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 09:40:26,529 main DEBUG Took 0.013536 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 09:40:26,530 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 09:40:26,530 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 09:40:26,532 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 09:40:26,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,543 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 09:40:26,562 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,566 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,568 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,569 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,571 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,571 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,572 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,572 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,573 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,573 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,574 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,574 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,575 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,575 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,576 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,577 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,577 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,578 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:40:26,578 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,578 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 09:40:26,580 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:40:26,582 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 09:40:26,585 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 09:40:26,585 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 09:40:26,587 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 09:40:26,587 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 09:40:26,598 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 09:40:26,601 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 09:40:26,604 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 09:40:26,605 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 09:40:26,606 main DEBUG createAppenders(={Console}) 2024-11-17 09:40:26,607 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 09:40:26,607 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 09:40:26,608 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 09:40:26,608 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 09:40:26,609 main DEBUG OutputStream closed 2024-11-17 09:40:26,609 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 09:40:26,610 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 09:40:26,610 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 09:40:26,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 09:40:26,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 09:40:26,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 09:40:26,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 09:40:26,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 09:40:26,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 09:40:26,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 09:40:26,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 09:40:26,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 09:40:26,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 09:40:26,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 09:40:26,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 09:40:26,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 09:40:26,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 09:40:26,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 09:40:26,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 09:40:26,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 09:40:26,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 09:40:26,734 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 09:40:26,735 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 09:40:26,735 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 09:40:26,736 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T09:40:27,139 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec 2024-11-17 09:40:27,145 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 09:40:27,145 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T09:40:27,159 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-17T09:40:27,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T09:40:27,230 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5, deleteOnExit=true 2024-11-17T09:40:27,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T09:40:27,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/test.cache.data in system properties and HBase conf 2024-11-17T09:40:27,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T09:40:27,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/hadoop.log.dir in system properties and HBase conf 2024-11-17T09:40:27,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T09:40:27,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T09:40:27,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T09:40:27,365 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T09:40:27,528 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:40:27,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T09:40:27,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T09:40:27,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T09:40:27,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T09:40:27,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T09:40:27,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T09:40:27,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T09:40:27,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T09:40:27,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T09:40:27,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/nfs.dump.dir in system properties and HBase conf 2024-11-17T09:40:27,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/java.io.tmpdir in system properties and HBase conf 2024-11-17T09:40:27,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T09:40:27,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T09:40:27,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T09:40:28,505 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T09:40:28,590 INFO [Time-limited test {}] log.Log(170): Logging initialized @2971ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T09:40:28,674 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T09:40:28,744 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T09:40:28,767 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T09:40:28,768 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T09:40:28,769 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T09:40:28,785 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T09:40:28,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/hadoop.log.dir/,AVAILABLE} 2024-11-17T09:40:28,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T09:40:29,015 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/java.io.tmpdir/jetty-localhost-42205-hadoop-hdfs-3_4_1-tests_jar-_-any-9277698844667538238/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T09:40:29,023 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42205} 2024-11-17T09:40:29,023 INFO [Time-limited test {}] server.Server(415): Started @3405ms 2024-11-17T09:40:29,468 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T09:40:29,478 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T09:40:29,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T09:40:29,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T09:40:29,480 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T09:40:29,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/hadoop.log.dir/,AVAILABLE} 2024-11-17T09:40:29,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T09:40:29,610 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/java.io.tmpdir/jetty-localhost-39955-hadoop-hdfs-3_4_1-tests_jar-_-any-5595236244955967047/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T09:40:29,611 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39955} 2024-11-17T09:40:29,611 INFO [Time-limited test {}] server.Server(415): Started @3993ms 2024-11-17T09:40:29,670 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T09:40:30,315 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5/data/data1/current/BP-123921581-172.17.0.2-1731836428229/current, will proceed with Du for space computation calculation, 2024-11-17T09:40:30,318 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5/data/data2/current/BP-123921581-172.17.0.2-1731836428229/current, will proceed with Du for space computation calculation, 2024-11-17T09:40:30,366 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T09:40:30,467 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xadf1a0974c69ce92 with lease ID 0x221748a36471ee22: Processing first storage report for DS-effbccf4-55fa-435a-b66f-15e7530bf79e from datanode DatanodeRegistration(127.0.0.1:38725, datanodeUuid=7f96f725-ead6-4d8e-9771-bfbf62a57c52, infoPort=40639, infoSecurePort=0, ipcPort=43383, storageInfo=lv=-57;cid=testClusterID;nsid=512031185;c=1731836428229) 2024-11-17T09:40:30,469 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xadf1a0974c69ce92 with lease ID 0x221748a36471ee22: from storage DS-effbccf4-55fa-435a-b66f-15e7530bf79e node DatanodeRegistration(127.0.0.1:38725, datanodeUuid=7f96f725-ead6-4d8e-9771-bfbf62a57c52, infoPort=40639, infoSecurePort=0, ipcPort=43383, storageInfo=lv=-57;cid=testClusterID;nsid=512031185;c=1731836428229), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T09:40:30,470 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xadf1a0974c69ce92 with lease ID 0x221748a36471ee22: Processing first storage report for DS-bda27fea-9e8f-4e0f-a6ed-0a8c63f2df36 from datanode DatanodeRegistration(127.0.0.1:38725, datanodeUuid=7f96f725-ead6-4d8e-9771-bfbf62a57c52, infoPort=40639, infoSecurePort=0, ipcPort=43383, storageInfo=lv=-57;cid=testClusterID;nsid=512031185;c=1731836428229) 2024-11-17T09:40:30,470 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xadf1a0974c69ce92 with lease ID 0x221748a36471ee22: from storage DS-bda27fea-9e8f-4e0f-a6ed-0a8c63f2df36 node DatanodeRegistration(127.0.0.1:38725, datanodeUuid=7f96f725-ead6-4d8e-9771-bfbf62a57c52, infoPort=40639, infoSecurePort=0, ipcPort=43383, storageInfo=lv=-57;cid=testClusterID;nsid=512031185;c=1731836428229), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T09:40:30,481 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec 2024-11-17T09:40:30,601 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5/zookeeper_0, clientPort=60879, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T09:40:30,619 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60879 2024-11-17T09:40:30,634 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T09:40:30,638 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T09:40:30,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741825_1001 (size=7) 2024-11-17T09:40:31,328 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be with version=8 2024-11-17T09:40:31,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/hbase-staging 2024-11-17T09:40:31,433 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T09:40:31,692 INFO [Time-limited test {}] client.ConnectionUtils(128): master/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:40:31,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:40:31,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:40:31,708 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:40:31,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:40:31,708 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:40:31,876 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T09:40:31,956 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T09:40:31,966 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T09:40:31,970 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:40:31,998 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1547 (auto-detected) 2024-11-17T09:40:31,999 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T09:40:32,019 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44533 2024-11-17T09:40:32,042 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44533 connecting to ZooKeeper ensemble=127.0.0.1:60879 2024-11-17T09:40:32,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:445330x0, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:40:32,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44533-0x101254cb3790000 connected 2024-11-17T09:40:32,126 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T09:40:32,128 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T09:40:32,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:40:32,145 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be, hbase.cluster.distributed=false 2024-11-17T09:40:32,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:40:32,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44533 2024-11-17T09:40:32,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44533 2024-11-17T09:40:32,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44533 2024-11-17T09:40:32,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44533 2024-11-17T09:40:32,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44533 2024-11-17T09:40:32,340 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:40:32,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:40:32,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:40:32,343 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:40:32,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:40:32,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:40:32,347 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T09:40:32,350 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:40:32,351 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40671 2024-11-17T09:40:32,353 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40671 connecting to ZooKeeper ensemble=127.0.0.1:60879 2024-11-17T09:40:32,354 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T09:40:32,357 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T09:40:32,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406710x0, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:40:32,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406710x0, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:40:32,368 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40671-0x101254cb3790001 connected 2024-11-17T09:40:32,370 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T09:40:32,385 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T09:40:32,388 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T09:40:32,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:40:32,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40671 2024-11-17T09:40:32,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40671 2024-11-17T09:40:32,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40671 2024-11-17T09:40:32,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40671 2024-11-17T09:40:32,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40671 2024-11-17T09:40:32,418 DEBUG [M:0;70e37c475432:44533 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;70e37c475432:44533 2024-11-17T09:40:32,419 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/70e37c475432,44533,1731836431485 2024-11-17T09:40:32,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:40:32,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:40:32,433 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/70e37c475432,44533,1731836431485 2024-11-17T09:40:32,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T09:40:32,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:32,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:32,458 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T09:40:32,459 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/70e37c475432,44533,1731836431485 from backup master directory 2024-11-17T09:40:32,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:40:32,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/70e37c475432,44533,1731836431485 2024-11-17T09:40:32,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:40:32,463 WARN [master/70e37c475432:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:40:32,463 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=70e37c475432,44533,1731836431485 2024-11-17T09:40:32,466 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T09:40:32,468 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T09:40:32,536 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/hbase.id] with ID: 10613bf0-d77c-4efa-912a-b479d3d211a8 2024-11-17T09:40:32,537 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/.tmp/hbase.id 2024-11-17T09:40:32,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741826_1002 (size=42) 2024-11-17T09:40:32,950 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/.tmp/hbase.id]:[hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/hbase.id] 2024-11-17T09:40:32,999 INFO [master/70e37c475432:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T09:40:33,004 INFO [master/70e37c475432:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T09:40:33,023 INFO [master/70e37c475432:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-17T09:40:33,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:33,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:33,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741827_1003 (size=196) 2024-11-17T09:40:33,060 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:40:33,063 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T09:40:33,082 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:33,086 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:40:33,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741828_1004 (size=1189) 2024-11-17T09:40:33,141 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store 2024-11-17T09:40:33,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741829_1005 (size=34) 2024-11-17T09:40:33,568 INFO [master/70e37c475432:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T09:40:33,571 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:40:33,573 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T09:40:33,573 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:40:33,573 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:40:33,574 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T09:40:33,575 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:40:33,575 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:40:33,576 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731836433572Disabling compacts and flushes for region at 1731836433572Disabling writes for close at 1731836433574 (+2 ms)Writing region close event to WAL at 1731836433575 (+1 ms)Closed at 1731836433575 2024-11-17T09:40:33,579 WARN [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/.initializing 2024-11-17T09:40:33,579 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/WALs/70e37c475432,44533,1731836431485 2024-11-17T09:40:33,588 INFO [master/70e37c475432:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:40:33,603 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=70e37c475432%2C44533%2C1731836431485, suffix=, logDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/WALs/70e37c475432,44533,1731836431485, archiveDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/oldWALs, maxLogs=10 2024-11-17T09:40:33,637 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/WALs/70e37c475432,44533,1731836431485/70e37c475432%2C44533%2C1731836431485.1731836433609, exclude list is [], retry=0 2024-11-17T09:40:33,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38725,DS-effbccf4-55fa-435a-b66f-15e7530bf79e,DISK] 2024-11-17T09:40:33,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T09:40:33,710 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/WALs/70e37c475432,44533,1731836431485/70e37c475432%2C44533%2C1731836431485.1731836433609 2024-11-17T09:40:33,711 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40639:40639)] 2024-11-17T09:40:33,712 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:40:33,712 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:40:33,717 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,718 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T09:40:33,796 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:33,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:33,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,803 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T09:40:33,803 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:33,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:40:33,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T09:40:33,809 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:33,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:40:33,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T09:40:33,813 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:33,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:40:33,815 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,818 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,820 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,825 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,826 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,829 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T09:40:33,833 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:40:33,838 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:40:33,839 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64174188, jitterRate=-0.043730080127716064}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T09:40:33,844 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731836433733Initializing all the Stores at 1731836433736 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836433736Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836433737 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836433737Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836433737Cleaning up temporary data from old regions at 1731836433826 (+89 ms)Region opened successfully at 1731836433844 (+18 ms) 2024-11-17T09:40:33,846 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T09:40:33,884 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2972663e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:40:33,921 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T09:40:33,934 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T09:40:33,934 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T09:40:33,937 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T09:40:33,939 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T09:40:33,944 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-17T09:40:33,945 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T09:40:33,975 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T09:40:33,986 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T09:40:33,990 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T09:40:33,994 INFO [master/70e37c475432:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T09:40:33,996 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T09:40:33,998 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T09:40:34,000 INFO [master/70e37c475432:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T09:40:34,005 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T09:40:34,007 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T09:40:34,009 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T09:40:34,011 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T09:40:34,037 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T09:40:34,040 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T09:40:34,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:40:34,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:40:34,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,049 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=70e37c475432,44533,1731836431485, sessionid=0x101254cb3790000, setting cluster-up flag (Was=false) 2024-11-17T09:40:34,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,078 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T09:40:34,081 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=70e37c475432,44533,1731836431485 2024-11-17T09:40:34,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,094 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T09:40:34,095 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=70e37c475432,44533,1731836431485 2024-11-17T09:40:34,102 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T09:40:34,183 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T09:40:34,195 INFO [master/70e37c475432:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T09:40:34,203 INFO [master/70e37c475432:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T09:40:34,207 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(746): ClusterId : 10613bf0-d77c-4efa-912a-b479d3d211a8 2024-11-17T09:40:34,210 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T09:40:34,211 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 70e37c475432,44533,1731836431485 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T09:40:34,218 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T09:40:34,218 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T09:40:34,221 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:40:34,221 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:40:34,221 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:40:34,221 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T09:40:34,221 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:40:34,222 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/70e37c475432:0, corePoolSize=10, maxPoolSize=10 2024-11-17T09:40:34,222 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,222 DEBUG [RS:0;70e37c475432:40671 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86d570f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:40:34,222 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:40:34,222 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,231 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731836464231 2024-11-17T09:40:34,233 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T09:40:34,234 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T09:40:34,235 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:40:34,236 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T09:40:34,239 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T09:40:34,240 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T09:40:34,240 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T09:40:34,240 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T09:40:34,243 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:34,241 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,243 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T09:40:34,245 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T09:40:34,245 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;70e37c475432:40671 2024-11-17T09:40:34,246 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T09:40:34,247 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T09:40:34,249 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T09:40:34,250 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T09:40:34,250 INFO [RS:0;70e37c475432:40671 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T09:40:34,250 INFO [RS:0;70e37c475432:40671 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T09:40:34,250 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T09:40:34,252 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836434251,5,FailOnTimeoutGroup] 2024-11-17T09:40:34,252 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836434252,5,FailOnTimeoutGroup] 2024-11-17T09:40:34,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741831_1007 (size=1321) 2024-11-17T09:40:34,252 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,253 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T09:40:34,254 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,44533,1731836431485 with port=40671, startcode=1731836432285 2024-11-17T09:40:34,254 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,254 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,255 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T09:40:34,255 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:40:34,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741832_1008 (size=32) 2024-11-17T09:40:34,270 DEBUG [RS:0;70e37c475432:40671 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T09:40:34,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44763, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T09:40:34,315 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44533 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 70e37c475432,40671,1731836432285 2024-11-17T09:40:34,317 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44533 {}] master.ServerManager(517): Registering regionserver=70e37c475432,40671,1731836432285 2024-11-17T09:40:34,332 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:40:34,333 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43475 2024-11-17T09:40:34,333 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T09:40:34,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:40:34,338 DEBUG [RS:0;70e37c475432:40671 {}] zookeeper.ZKUtil(111): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/70e37c475432,40671,1731836432285 2024-11-17T09:40:34,338 WARN [RS:0;70e37c475432:40671 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:40:34,338 INFO [RS:0;70e37c475432:40671 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:40:34,338 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285 2024-11-17T09:40:34,340 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [70e37c475432,40671,1731836432285] 2024-11-17T09:40:34,364 INFO [RS:0;70e37c475432:40671 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T09:40:34,377 INFO [RS:0;70e37c475432:40671 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T09:40:34,382 INFO [RS:0;70e37c475432:40671 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T09:40:34,382 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,383 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T09:40:34,389 INFO [RS:0;70e37c475432:40671 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T09:40:34,391 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,391 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,391 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,391 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,392 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,392 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,392 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:40:34,392 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,393 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,393 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,393 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,393 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,394 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:40:34,394 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:40:34,394 DEBUG [RS:0;70e37c475432:40671 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:40:34,395 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,395 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,396 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,396 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,396 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,396 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40671,1731836432285-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:40:34,417 INFO [RS:0;70e37c475432:40671 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T09:40:34,419 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40671,1731836432285-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,419 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,419 INFO [RS:0;70e37c475432:40671 {}] regionserver.Replication(171): 70e37c475432,40671,1731836432285 started 2024-11-17T09:40:34,439 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:34,439 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1482): Serving as 70e37c475432,40671,1731836432285, RpcServer on 70e37c475432/172.17.0.2:40671, sessionid=0x101254cb3790001 2024-11-17T09:40:34,440 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T09:40:34,440 DEBUG [RS:0;70e37c475432:40671 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 70e37c475432,40671,1731836432285 2024-11-17T09:40:34,440 DEBUG [RS:0;70e37c475432:40671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,40671,1731836432285' 2024-11-17T09:40:34,440 DEBUG [RS:0;70e37c475432:40671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T09:40:34,441 DEBUG [RS:0;70e37c475432:40671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T09:40:34,442 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T09:40:34,442 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T09:40:34,442 DEBUG [RS:0;70e37c475432:40671 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 70e37c475432,40671,1731836432285 2024-11-17T09:40:34,442 DEBUG [RS:0;70e37c475432:40671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,40671,1731836432285' 2024-11-17T09:40:34,442 DEBUG [RS:0;70e37c475432:40671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T09:40:34,443 DEBUG [RS:0;70e37c475432:40671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T09:40:34,443 DEBUG [RS:0;70e37c475432:40671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T09:40:34,444 INFO [RS:0;70e37c475432:40671 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T09:40:34,444 INFO [RS:0;70e37c475432:40671 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T09:40:34,549 INFO [RS:0;70e37c475432:40671 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:40:34,553 INFO [RS:0;70e37c475432:40671 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=70e37c475432%2C40671%2C1731836432285, suffix=, logDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285, archiveDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/oldWALs, maxLogs=32 2024-11-17T09:40:34,570 DEBUG [RS:0;70e37c475432:40671 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285/70e37c475432%2C40671%2C1731836432285.1731836434555, exclude list is [], retry=0 2024-11-17T09:40:34,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38725,DS-effbccf4-55fa-435a-b66f-15e7530bf79e,DISK] 2024-11-17T09:40:34,578 INFO [RS:0;70e37c475432:40671 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285/70e37c475432%2C40671%2C1731836432285.1731836434555 2024-11-17T09:40:34,579 DEBUG [RS:0;70e37c475432:40671 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40639:40639)] 2024-11-17T09:40:34,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:40:34,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T09:40:34,672 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T09:40:34,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:34,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:34,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T09:40:34,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T09:40:34,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:34,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:34,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T09:40:34,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T09:40:34,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:34,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:34,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T09:40:34,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T09:40:34,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:34,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:34,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T09:40:34,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740 2024-11-17T09:40:34,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740 2024-11-17T09:40:34,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T09:40:34,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T09:40:34,689 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:40:34,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T09:40:34,694 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:40:34,695 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72794725, jitterRate=0.08472593128681183}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:40:34,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731836434667Initializing all the Stores at 1731836434669 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836434669Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836434669Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836434669Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836434669Cleaning up temporary data from old regions at 1731836434688 (+19 ms)Region opened successfully at 1731836434697 (+9 ms) 2024-11-17T09:40:34,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T09:40:34,697 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T09:40:34,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T09:40:34,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T09:40:34,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T09:40:34,699 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T09:40:34,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731836434697Disabling compacts and flushes for region at 1731836434697Disabling writes for close at 1731836434698 (+1 ms)Writing region close event to WAL at 1731836434698Closed at 1731836434699 (+1 ms) 2024-11-17T09:40:34,703 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:40:34,703 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T09:40:34,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T09:40:34,720 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T09:40:34,723 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T09:40:34,875 DEBUG [70e37c475432:44533 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T09:40:34,888 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:40:34,895 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 70e37c475432,40671,1731836432285, state=OPENING 2024-11-17T09:40:34,900 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T09:40:34,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:40:34,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:40:34,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:40:34,905 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T09:40:34,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:40:35,085 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T09:40:35,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34399, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T09:40:35,101 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T09:40:35,101 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:40:35,101 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T09:40:35,105 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=70e37c475432%2C40671%2C1731836432285.meta, suffix=.meta, logDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285, archiveDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/oldWALs, maxLogs=32 2024-11-17T09:40:35,121 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285/70e37c475432%2C40671%2C1731836432285.meta.1731836435106.meta, exclude list is [], retry=0 2024-11-17T09:40:35,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38725,DS-effbccf4-55fa-435a-b66f-15e7530bf79e,DISK] 2024-11-17T09:40:35,128 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285/70e37c475432%2C40671%2C1731836432285.meta.1731836435106.meta 2024-11-17T09:40:35,129 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40639:40639)] 2024-11-17T09:40:35,129 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:40:35,131 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T09:40:35,134 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T09:40:35,138 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T09:40:35,143 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T09:40:35,143 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:40:35,143 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T09:40:35,143 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T09:40:35,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T09:40:35,148 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T09:40:35,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:35,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:35,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T09:40:35,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T09:40:35,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:35,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:35,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T09:40:35,153 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T09:40:35,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:35,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:35,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T09:40:35,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T09:40:35,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:35,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:40:35,157 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T09:40:35,158 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740 2024-11-17T09:40:35,161 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740 2024-11-17T09:40:35,163 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T09:40:35,163 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T09:40:35,164 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:40:35,167 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T09:40:35,169 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71083933, jitterRate=0.05923314392566681}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:40:35,169 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T09:40:35,171 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731836435144Writing region info on filesystem at 1731836435144Initializing all the Stores at 1731836435146 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836435146Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836435146Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836435146Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836435146Cleaning up temporary data from old regions at 1731836435163 (+17 ms)Running coprocessor post-open hooks at 1731836435169 (+6 ms)Region opened successfully at 1731836435171 (+2 ms) 2024-11-17T09:40:35,178 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731836435076 2024-11-17T09:40:35,189 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T09:40:35,190 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T09:40:35,191 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:40:35,193 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 70e37c475432,40671,1731836432285, state=OPEN 2024-11-17T09:40:35,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:40:35,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:40:35,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:40:35,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:40:35,205 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:35,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T09:40:35,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=70e37c475432,40671,1731836432285 in 299 msec 2024-11-17T09:40:35,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T09:40:35,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 503 msec 2024-11-17T09:40:35,222 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:40:35,222 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T09:40:35,244 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:35,246 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:35,268 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:35,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:35,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1590 sec 2024-11-17T09:40:35,295 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731836435295, completionTime=-1 2024-11-17T09:40:35,298 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T09:40:35,298 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T09:40:35,354 INFO [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-17T09:40:35,354 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731836495354 2024-11-17T09:40:35,354 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731836555354 2024-11-17T09:40:35,354 INFO [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 55 msec 2024-11-17T09:40:35,356 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,44533,1731836431485-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:35,357 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,44533,1731836431485-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:35,357 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,44533,1731836431485-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:35,358 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-70e37c475432:44533, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:35,359 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:35,359 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:35,366 DEBUG [master/70e37c475432:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T09:40:35,386 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.922sec 2024-11-17T09:40:35,387 INFO [master/70e37c475432:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T09:40:35,388 INFO [master/70e37c475432:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T09:40:35,389 INFO [master/70e37c475432:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T09:40:35,390 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T09:40:35,390 INFO [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T09:40:35,391 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,44533,1731836431485-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:40:35,391 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,44533,1731836431485-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T09:40:35,399 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T09:40:35,400 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T09:40:35,400 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,44533,1731836431485-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:40:35,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:35,419 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T09:40:35,419 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T09:40:35,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:35,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:35,432 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f93b2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:35,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:35,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:35,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:35,446 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:35,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:35,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=70e37c475432,44533,1731836431485 2024-11-17T09:40:35,487 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=3711 2024-11-17T09:40:35,495 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:40:35,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,44533,1731836431485 2024-11-17T09:40:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52246f65 2024-11-17T09:40:35,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:40:35,506 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:40:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:40:35,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:40:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T09:40:35,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:40:35,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T09:40:35,526 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:35,529 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:40:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:40:35,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741835_1011 (size=1061) 2024-11-17T09:40:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:40:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:40:35,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:40:35,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741836_1012 (size=53) 2024-11-17T09:40:35,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:40:35,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c0fd42d9906a7b067f93ee39238302f5, disabling compactions & flushes 2024-11-17T09:40:35,955 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:35,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:35,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. after waiting 0 ms 2024-11-17T09:40:35,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:35,956 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:35,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0fd42d9906a7b067f93ee39238302f5: Waiting for close lock at 1731836435955Disabling compacts and flushes for region at 1731836435955Disabling writes for close at 1731836435956 (+1 ms)Writing region close event to WAL at 1731836435956Closed at 1731836435956 2024-11-17T09:40:35,958 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:40:35,963 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731836435958"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836435958"}]},"ts":"1731836435958"} 2024-11-17T09:40:35,968 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:40:35,969 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:40:35,972 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836435970"}]},"ts":"1731836435970"} 2024-11-17T09:40:35,976 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T09:40:35,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, ASSIGN}] 2024-11-17T09:40:35,980 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, ASSIGN 2024-11-17T09:40:35,982 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, ASSIGN; state=OFFLINE, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=false 2024-11-17T09:40:36,134 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0fd42d9906a7b067f93ee39238302f5, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:40:36,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, ASSIGN because future has completed 2024-11-17T09:40:36,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:40:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:40:36,300 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:36,301 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:40:36,301 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,301 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:40:36,302 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,302 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,304 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,307 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:40:36,307 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0fd42d9906a7b067f93ee39238302f5 columnFamilyName A 2024-11-17T09:40:36,308 DEBUG [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:36,309 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.HStore(327): Store=c0fd42d9906a7b067f93ee39238302f5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:40:36,309 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,310 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:40:36,311 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0fd42d9906a7b067f93ee39238302f5 columnFamilyName B 2024-11-17T09:40:36,311 DEBUG [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:36,312 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.HStore(327): Store=c0fd42d9906a7b067f93ee39238302f5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:40:36,312 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,314 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:40:36,314 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0fd42d9906a7b067f93ee39238302f5 columnFamilyName C 2024-11-17T09:40:36,314 DEBUG [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:40:36,315 INFO [StoreOpener-c0fd42d9906a7b067f93ee39238302f5-1 {}] regionserver.HStore(327): Store=c0fd42d9906a7b067f93ee39238302f5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:40:36,315 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:36,315 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,317 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,318 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,319 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,320 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,321 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:40:36,323 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,327 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:40:36,328 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c0fd42d9906a7b067f93ee39238302f5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65617413, jitterRate=-0.02222435176372528}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:40:36,329 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,330 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c0fd42d9906a7b067f93ee39238302f5: Running coprocessor pre-open hook at 1731836436302Writing region info on filesystem at 1731836436302Initializing all the Stores at 1731836436303 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836436303Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836436304 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836436304Cleaning up temporary data from old regions at 1731836436320 (+16 ms)Running coprocessor post-open hooks at 1731836436329 (+9 ms)Region opened successfully at 1731836436329 2024-11-17T09:40:36,332 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., pid=6, masterSystemTime=1731836436294 2024-11-17T09:40:36,335 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:36,335 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:36,336 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0fd42d9906a7b067f93ee39238302f5, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:40:36,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:40:36,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T09:40:36,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 in 203 msec 2024-11-17T09:40:36,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T09:40:36,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, ASSIGN in 368 msec 2024-11-17T09:40:36,352 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:40:36,352 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836436352"}]},"ts":"1731836436352"} 2024-11-17T09:40:36,355 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T09:40:36,356 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:40:36,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 842 msec 2024-11-17T09:40:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:40:36,672 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:36,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c32821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,682 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29446fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e077d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,687 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,693 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:40:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,44533,1731836431485 2024-11-17T09:40:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d4e458b 2024-11-17T09:40:36,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:40:36,698 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:40:36,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd90ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,703 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,709 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,713 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,716 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,724 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,727 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dac934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,729 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,730 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e3ab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,733 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8dae85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ddbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,737 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b105800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,739 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e234a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a48b566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44731cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,745 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@870009d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,748 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d95629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,751 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3837915e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:40:36,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:40:36,754 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:40:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:40:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:40:36,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e873e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:40:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:40:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:36,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:40:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21616163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:40:36,760 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,760 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,760 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,761 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,761 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,766 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,767 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,767 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,770 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:40:36,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:40:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:40:36,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:40:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,782 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:36,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T09:40:36,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:40:36,793 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T09:40:36,795 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:36,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:36,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:36,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:40:36,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:36,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T09:40:36,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:36,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:36,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:36,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:36,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:36,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836496977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836496988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836496990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/bd47a1529721499887bb51a3b6f0f618 is 50, key is test_row_0/A:col10/1731836436831/Put/seqid=0 2024-11-17T09:40:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836497006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836497015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741837_1013 (size=14341) 2024-11-17T09:40:37,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/bd47a1529721499887bb51a3b6f0f618 2024-11-17T09:40:37,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T09:40:37,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/68a9cd6058c14bd6a7ae45749949c4db is 50, key is test_row_0/B:col10/1731836436831/Put/seqid=0 2024-11-17T09:40:37,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:37,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:37,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836497153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836497153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836497151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741838_1014 (size=12001) 2024-11-17T09:40:37,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836497152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836497153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:37,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:37,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836497371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836497381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836497382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836497385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836497391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T09:40:37,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:37,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:37,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/68a9cd6058c14bd6a7ae45749949c4db 2024-11-17T09:40:37,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6334806e250449128af8558d43619f98 is 50, key is test_row_0/C:col10/1731836436831/Put/seqid=0 2024-11-17T09:40:37,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:37,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:37,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741839_1015 (size=12001) 2024-11-17T09:40:37,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6334806e250449128af8558d43619f98 2024-11-17T09:40:37,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/bd47a1529721499887bb51a3b6f0f618 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/bd47a1529721499887bb51a3b6f0f618 2024-11-17T09:40:37,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/bd47a1529721499887bb51a3b6f0f618, entries=200, sequenceid=14, filesize=14.0 K 2024-11-17T09:40:37,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836497681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,690 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-17T09:40:37,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/68a9cd6058c14bd6a7ae45749949c4db as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/68a9cd6058c14bd6a7ae45749949c4db 2024-11-17T09:40:37,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836497690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836497690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/68a9cd6058c14bd6a7ae45749949c4db, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T09:40:37,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6334806e250449128af8558d43619f98 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6334806e250449128af8558d43619f98 2024-11-17T09:40:37,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6334806e250449128af8558d43619f98, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T09:40:37,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c0fd42d9906a7b067f93ee39238302f5 in 847ms, sequenceid=14, compaction requested=false 2024-11-17T09:40:37,733 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T09:40:37,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:37,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836436979, max timestamp=1731836437736Num uniques 859; 2024-11-17T09:40:37,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:40:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:37,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-17T09:40:37,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:37,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:37,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:37,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:37,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/6ccb974a4a944cae877de73a3df634fc is 50, key is test_row_0/A:col10/1731836436988/Put/seqid=0 2024-11-17T09:40:37,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:37,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741840_1016 (size=16681) 2024-11-17T09:40:37,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/6ccb974a4a944cae877de73a3df634fc 2024-11-17T09:40:37,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/3064624501454fa794812647c7a373c4 is 50, key is test_row_0/B:col10/1731836436988/Put/seqid=0 2024-11-17T09:40:37,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741841_1017 (size=12001) 2024-11-17T09:40:37,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/3064624501454fa794812647c7a373c4 2024-11-17T09:40:37,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6c937adae7ab421d9927d9a8a23e2117 is 50, key is test_row_0/C:col10/1731836436988/Put/seqid=0 2024-11-17T09:40:37,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741842_1018 (size=12001) 2024-11-17T09:40:37,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6c937adae7ab421d9927d9a8a23e2117 2024-11-17T09:40:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T09:40:37,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/6ccb974a4a944cae877de73a3df634fc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/6ccb974a4a944cae877de73a3df634fc 2024-11-17T09:40:37,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/6ccb974a4a944cae877de73a3df634fc, entries=250, sequenceid=36, filesize=16.3 K 2024-11-17T09:40:37,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/3064624501454fa794812647c7a373c4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3064624501454fa794812647c7a373c4 2024-11-17T09:40:37,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:37,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:37,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3064624501454fa794812647c7a373c4, entries=150, sequenceid=36, filesize=11.7 K 2024-11-17T09:40:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6c937adae7ab421d9927d9a8a23e2117 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6c937adae7ab421d9927d9a8a23e2117 2024-11-17T09:40:37,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836497971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:37,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6c937adae7ab421d9927d9a8a23e2117, entries=150, sequenceid=36, filesize=11.7 K 2024-11-17T09:40:37,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for c0fd42d9906a7b067f93ee39238302f5 in 224ms, sequenceid=36, compaction requested=false 2024-11-17T09:40:37,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:37,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:40:37,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:37,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:37,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:37,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:37,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:37,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:38,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/407dbac55ff046489681615d54ebdd93 is 50, key is test_row_0/A:col10/1731836437948/Put/seqid=0 2024-11-17T09:40:38,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741843_1019 (size=16681) 2024-11-17T09:40:38,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/407dbac55ff046489681615d54ebdd93 2024-11-17T09:40:38,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/75b26c5625ae4e899a2f81bead5ea937 is 50, key is test_row_0/B:col10/1731836437948/Put/seqid=0 2024-11-17T09:40:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741844_1020 (size=12001) 2024-11-17T09:40:38,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836498126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836498131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836498201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836498201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836498202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836498240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836498251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:38,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:38,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836498461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836498461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/75b26c5625ae4e899a2f81bead5ea937 2024-11-17T09:40:38,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/84458f29c7914e2c87b3c500c41324f0 is 50, key is test_row_0/C:col10/1731836437948/Put/seqid=0 2024-11-17T09:40:38,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741845_1021 (size=12001) 2024-11-17T09:40:38,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/84458f29c7914e2c87b3c500c41324f0 2024-11-17T09:40:38,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/407dbac55ff046489681615d54ebdd93 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/407dbac55ff046489681615d54ebdd93 2024-11-17T09:40:38,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/407dbac55ff046489681615d54ebdd93, entries=250, sequenceid=52, filesize=16.3 K 2024-11-17T09:40:38,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/75b26c5625ae4e899a2f81bead5ea937 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/75b26c5625ae4e899a2f81bead5ea937 2024-11-17T09:40:38,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/75b26c5625ae4e899a2f81bead5ea937, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T09:40:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/84458f29c7914e2c87b3c500c41324f0 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/84458f29c7914e2c87b3c500c41324f0 2024-11-17T09:40:38,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/84458f29c7914e2c87b3c500c41324f0, entries=150, sequenceid=52, filesize=11.7 K 2024-11-17T09:40:38,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 590ms, sequenceid=52, compaction requested=true 2024-11-17T09:40:38,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:38,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T09:40:38,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,589 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:40:38,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:38,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:38,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:38,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:38,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:38,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:38,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:38,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/fb4b91d221fc4672a0c39653d34cdd25 is 50, key is test_row_0/A:col10/1731836438097/Put/seqid=0 2024-11-17T09:40:38,608 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:38,611 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:38,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:38,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:38,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:38,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:38,615 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:38,615 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:38,616 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:38,616 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:38,621 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,621 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,622 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/bd47a1529721499887bb51a3b6f0f618, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/6ccb974a4a944cae877de73a3df634fc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/407dbac55ff046489681615d54ebdd93] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=46.6 K 2024-11-17T09:40:38,622 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/68a9cd6058c14bd6a7ae45749949c4db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3064624501454fa794812647c7a373c4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/75b26c5625ae4e899a2f81bead5ea937] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=35.2 K 2024-11-17T09:40:38,623 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd47a1529721499887bb51a3b6f0f618, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731836436831 2024-11-17T09:40:38,623 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a9cd6058c14bd6a7ae45749949c4db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731836436831 2024-11-17T09:40:38,625 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ccb974a4a944cae877de73a3df634fc, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836436979 2024-11-17T09:40:38,625 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3064624501454fa794812647c7a373c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836436988 2024-11-17T09:40:38,626 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b26c5625ae4e899a2f81bead5ea937, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731836437948 2024-11-17T09:40:38,626 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 407dbac55ff046489681615d54ebdd93, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731836437900 2024-11-17T09:40:38,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741846_1022 (size=12001) 2024-11-17T09:40:38,665 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:38,666 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/06b9fc4ece5f4be58352bd799ee3d9eb is 50, key is test_row_0/B:col10/1731836437948/Put/seqid=0 2024-11-17T09:40:38,676 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#11 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:38,677 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/0e32b856b21d4ab7b882285ed9b08b0a is 50, key is test_row_0/A:col10/1731836437948/Put/seqid=0 2024-11-17T09:40:38,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741847_1023 (size=12104) 2024-11-17T09:40:38,722 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/06b9fc4ece5f4be58352bd799ee3d9eb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/06b9fc4ece5f4be58352bd799ee3d9eb 2024-11-17T09:40:38,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741848_1024 (size=12104) 2024-11-17T09:40:38,755 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 06b9fc4ece5f4be58352bd799ee3d9eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:38,755 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:38,758 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836438610; duration=0sec 2024-11-17T09:40:38,759 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:38,759 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:38,759 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:38,762 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:38,762 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:38,762 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:38,762 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6334806e250449128af8558d43619f98, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6c937adae7ab421d9927d9a8a23e2117, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/84458f29c7914e2c87b3c500c41324f0] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=35.2 K 2024-11-17T09:40:38,764 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6334806e250449128af8558d43619f98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731836436831 2024-11-17T09:40:38,765 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c937adae7ab421d9927d9a8a23e2117, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836436988 2024-11-17T09:40:38,766 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 84458f29c7914e2c87b3c500c41324f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731836437948 2024-11-17T09:40:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:38,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:38,790 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:38,792 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/8d13a00fb7bb496baac3d6693bf57bc8 is 50, key is test_row_0/C:col10/1731836437948/Put/seqid=0 2024-11-17T09:40:38,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741849_1025 (size=12104) 2024-11-17T09:40:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836498851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836498852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T09:40:38,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836498974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:38,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:38,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836498974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:38,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,045 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/fb4b91d221fc4672a0c39653d34cdd25 2024-11-17T09:40:39,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d4abe95d49464ca5807674ccce103302 is 50, key is test_row_0/B:col10/1731836438097/Put/seqid=0 2024-11-17T09:40:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741850_1026 (size=12001) 2024-11-17T09:40:39,082 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d4abe95d49464ca5807674ccce103302 2024-11-17T09:40:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/ab0fa5a7da5d48ac9d670e029b6cd66b is 50, key is test_row_0/C:col10/1731836438097/Put/seqid=0 2024-11-17T09:40:39,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741851_1027 (size=12001) 2024-11-17T09:40:39,150 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/0e32b856b21d4ab7b882285ed9b08b0a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0e32b856b21d4ab7b882285ed9b08b0a 2024-11-17T09:40:39,166 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 0e32b856b21d4ab7b882285ed9b08b0a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:39,166 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:39,167 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836438583; duration=0sec 2024-11-17T09:40:39,168 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:39,168 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:39,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836499180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836499191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836499220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:39,227 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836499230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:39,234 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836499230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:39,241 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,256 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/8d13a00fb7bb496baac3d6693bf57bc8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8d13a00fb7bb496baac3d6693bf57bc8 2024-11-17T09:40:39,269 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into 8d13a00fb7bb496baac3d6693bf57bc8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:39,269 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:39,269 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836438614; duration=0sec 2024-11-17T09:40:39,270 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:39,270 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836499500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:39,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836499500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:39,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:39,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:39,529 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/ab0fa5a7da5d48ac9d670e029b6cd66b 2024-11-17T09:40:39,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/fb4b91d221fc4672a0c39653d34cdd25 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/fb4b91d221fc4672a0c39653d34cdd25 2024-11-17T09:40:39,560 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/fb4b91d221fc4672a0c39653d34cdd25, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T09:40:39,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d4abe95d49464ca5807674ccce103302 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4abe95d49464ca5807674ccce103302 2024-11-17T09:40:39,588 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4abe95d49464ca5807674ccce103302, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T09:40:39,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/ab0fa5a7da5d48ac9d670e029b6cd66b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ab0fa5a7da5d48ac9d670e029b6cd66b 2024-11-17T09:40:39,608 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ab0fa5a7da5d48ac9d670e029b6cd66b, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T09:40:39,611 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 1022ms, sequenceid=68, compaction requested=false 2024-11-17T09:40:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T09:40:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T09:40:39,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T09:40:39,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8170 sec 2024-11-17T09:40:39,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.8390 sec 2024-11-17T09:40:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:40,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T09:40:40,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:40,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:40,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:40,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:40,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:40,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:40,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/eea55cb9f5e54cd48c92684630cca21c is 50, key is test_row_0/A:col10/1731836440014/Put/seqid=0 2024-11-17T09:40:40,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741852_1028 (size=16681) 2024-11-17T09:40:40,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836500092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836500093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836500200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836500200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836500411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836500420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/eea55cb9f5e54cd48c92684630cca21c 2024-11-17T09:40:40,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d45ab3e1dcd444628e3ab0981d4dfa94 is 50, key is test_row_0/B:col10/1731836440014/Put/seqid=0 2024-11-17T09:40:40,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741853_1029 (size=12001) 2024-11-17T09:40:40,658 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:40:40,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836500720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836500730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:40,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:40,759 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T09:40:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T09:40:40,960 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:40,961 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T09:40:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T09:40:40,967 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:40,969 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:40,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:40,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d45ab3e1dcd444628e3ab0981d4dfa94 2024-11-17T09:40:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T09:40:41,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/c618d5fb96c047c3832df10d50db03b8 is 50, key is test_row_0/C:col10/1731836440014/Put/seqid=0 2024-11-17T09:40:41,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T09:40:41,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741854_1030 (size=12001) 2024-11-17T09:40:41,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/c618d5fb96c047c3832df10d50db03b8 2024-11-17T09:40:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/eea55cb9f5e54cd48c92684630cca21c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/eea55cb9f5e54cd48c92684630cca21c 2024-11-17T09:40:41,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/eea55cb9f5e54cd48c92684630cca21c, entries=250, sequenceid=89, filesize=16.3 K 2024-11-17T09:40:41,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d45ab3e1dcd444628e3ab0981d4dfa94 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d45ab3e1dcd444628e3ab0981d4dfa94 2024-11-17T09:40:41,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d45ab3e1dcd444628e3ab0981d4dfa94, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T09:40:41,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/c618d5fb96c047c3832df10d50db03b8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/c618d5fb96c047c3832df10d50db03b8 2024-11-17T09:40:41,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/c618d5fb96c047c3832df10d50db03b8, entries=150, sequenceid=89, filesize=11.7 K 2024-11-17T09:40:41,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for c0fd42d9906a7b067f93ee39238302f5 in 1213ms, sequenceid=89, compaction requested=true 2024-11-17T09:40:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:41,232 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:41,232 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:41,234 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:41,234 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:41,235 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,235 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:41,235 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0e32b856b21d4ab7b882285ed9b08b0a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/fb4b91d221fc4672a0c39653d34cdd25, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/eea55cb9f5e54cd48c92684630cca21c] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=39.8 K 2024-11-17T09:40:41,235 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:41,235 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,235 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/06b9fc4ece5f4be58352bd799ee3d9eb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4abe95d49464ca5807674ccce103302, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d45ab3e1dcd444628e3ab0981d4dfa94] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=35.3 K 2024-11-17T09:40:41,236 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e32b856b21d4ab7b882285ed9b08b0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731836437948 2024-11-17T09:40:41,236 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b9fc4ece5f4be58352bd799ee3d9eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731836437948 2024-11-17T09:40:41,236 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb4b91d221fc4672a0c39653d34cdd25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731836438097 2024-11-17T09:40:41,237 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d4abe95d49464ca5807674ccce103302, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731836438097 2024-11-17T09:40:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:41,241 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d45ab3e1dcd444628e3ab0981d4dfa94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731836438845 2024-11-17T09:40:41,241 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea55cb9f5e54cd48c92684630cca21c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731836438826 2024-11-17T09:40:41,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:40:41,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:41,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:41,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:41,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:41,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:41,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:41,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/3e8d4651f35e4e138b437a7ecdde1879 is 50, key is test_row_0/A:col10/1731836440070/Put/seqid=0 2024-11-17T09:40:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T09:40:41,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T09:40:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,325 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:41,326 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/b834a7cbfcb044b8ac5dc958560f7636 is 50, key is test_row_0/B:col10/1731836440014/Put/seqid=0 2024-11-17T09:40:41,327 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:41,328 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/12305295f112466f8efea2150279f248 is 50, key is test_row_0/A:col10/1731836440014/Put/seqid=0 2024-11-17T09:40:41,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741855_1031 (size=12001) 2024-11-17T09:40:41,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/3e8d4651f35e4e138b437a7ecdde1879 2024-11-17T09:40:41,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836501338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836501340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836501339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836501344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836501348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/dac7252d90d244788276cc89b986b4e0 is 50, key is test_row_0/B:col10/1731836440070/Put/seqid=0 2024-11-17T09:40:41,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741856_1032 (size=12207) 2024-11-17T09:40:41,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741857_1033 (size=12207) 2024-11-17T09:40:41,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741858_1034 (size=12001) 2024-11-17T09:40:41,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836501461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836501461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836501461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836501462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T09:40:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836501462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T09:40:41,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T09:40:41,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836501680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836501680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836501681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836501681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836501681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:41,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T09:40:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:41,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:41,805 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/b834a7cbfcb044b8ac5dc958560f7636 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b834a7cbfcb044b8ac5dc958560f7636 2024-11-17T09:40:41,809 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/12305295f112466f8efea2150279f248 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/12305295f112466f8efea2150279f248 2024-11-17T09:40:41,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/dac7252d90d244788276cc89b986b4e0 2024-11-17T09:40:41,834 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into b834a7cbfcb044b8ac5dc958560f7636(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:41,834 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 12305295f112466f8efea2150279f248(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:41,834 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:41,835 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836441232; duration=0sec 2024-11-17T09:40:41,835 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:41,835 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836441232; duration=0sec 2024-11-17T09:40:41,835 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:41,835 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:41,835 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:41,835 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:41,835 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:41,837 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:41,837 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:41,837 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,838 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8d13a00fb7bb496baac3d6693bf57bc8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ab0fa5a7da5d48ac9d670e029b6cd66b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/c618d5fb96c047c3832df10d50db03b8] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=35.3 K 2024-11-17T09:40:41,839 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d13a00fb7bb496baac3d6693bf57bc8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731836437948 2024-11-17T09:40:41,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6ab16e43861c497bb6507d619e55bc21 is 50, key is test_row_0/C:col10/1731836440070/Put/seqid=0 2024-11-17T09:40:41,839 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab0fa5a7da5d48ac9d670e029b6cd66b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731836438097 2024-11-17T09:40:41,840 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c618d5fb96c047c3832df10d50db03b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731836438845 2024-11-17T09:40:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741859_1035 (size=12001) 2024-11-17T09:40:41,856 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:41,857 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/0afa7bd656ed426789c008934af1234f is 50, key is test_row_0/C:col10/1731836440014/Put/seqid=0 2024-11-17T09:40:41,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6ab16e43861c497bb6507d619e55bc21 2024-11-17T09:40:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741860_1036 (size=12207) 2024-11-17T09:40:41,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/3e8d4651f35e4e138b437a7ecdde1879 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3e8d4651f35e4e138b437a7ecdde1879 2024-11-17T09:40:41,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3e8d4651f35e4e138b437a7ecdde1879, entries=150, sequenceid=105, filesize=11.7 K 2024-11-17T09:40:41,893 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/0afa7bd656ed426789c008934af1234f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/0afa7bd656ed426789c008934af1234f 2024-11-17T09:40:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/dac7252d90d244788276cc89b986b4e0 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/dac7252d90d244788276cc89b986b4e0 2024-11-17T09:40:41,907 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into 0afa7bd656ed426789c008934af1234f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:41,907 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:41,907 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836441232; duration=0sec 2024-11-17T09:40:41,908 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:41,908 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:41,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/dac7252d90d244788276cc89b986b4e0, entries=150, sequenceid=105, filesize=11.7 K 2024-11-17T09:40:41,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6ab16e43861c497bb6507d619e55bc21 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6ab16e43861c497bb6507d619e55bc21 2024-11-17T09:40:41,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6ab16e43861c497bb6507d619e55bc21, entries=150, sequenceid=105, filesize=11.7 K 2024-11-17T09:40:41,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 693ms, sequenceid=105, compaction requested=false 2024-11-17T09:40:41,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:41,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T09:40:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:41,942 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:40:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/deae0acd305b49f4ba09b1a9e78a0a25 is 50, key is test_row_0/A:col10/1731836441338/Put/seqid=0 2024-11-17T09:40:41,953 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T09:40:41,954 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T09:40:41,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T09:40:41,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T09:40:41,956 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T09:40:41,956 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T09:40:41,958 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T09:40:41,958 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T09:40:41,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:41,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741861_1037 (size=12001) 2024-11-17T09:40:42,000 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/deae0acd305b49f4ba09b1a9e78a0a25 2024-11-17T09:40:42,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d84b7fbbc1a848f6b4e9003a90710690 is 50, key is test_row_0/B:col10/1731836441338/Put/seqid=0 2024-11-17T09:40:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836502018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836502020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836502029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836502027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836502032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741862_1038 (size=12001) 2024-11-17T09:40:42,056 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d84b7fbbc1a848f6b4e9003a90710690 2024-11-17T09:40:42,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/42a93ee4f0974f428843ee4dd58ddafb is 50, key is test_row_0/C:col10/1731836441338/Put/seqid=0 2024-11-17T09:40:42,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741863_1039 (size=12001) 2024-11-17T09:40:42,091 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/42a93ee4f0974f428843ee4dd58ddafb 2024-11-17T09:40:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T09:40:42,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/deae0acd305b49f4ba09b1a9e78a0a25 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/deae0acd305b49f4ba09b1a9e78a0a25 2024-11-17T09:40:42,110 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/deae0acd305b49f4ba09b1a9e78a0a25, entries=150, sequenceid=124, filesize=11.7 K 2024-11-17T09:40:42,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d84b7fbbc1a848f6b4e9003a90710690 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d84b7fbbc1a848f6b4e9003a90710690 2024-11-17T09:40:42,119 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d84b7fbbc1a848f6b4e9003a90710690, entries=150, sequenceid=124, filesize=11.7 K 2024-11-17T09:40:42,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/42a93ee4f0974f428843ee4dd58ddafb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/42a93ee4f0974f428843ee4dd58ddafb 2024-11-17T09:40:42,130 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/42a93ee4f0974f428843ee4dd58ddafb, entries=150, sequenceid=124, filesize=11.7 K 2024-11-17T09:40:42,132 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 190ms, sequenceid=124, compaction requested=true 2024-11-17T09:40:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:42,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T09:40:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T09:40:42,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T09:40:42,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1660 sec 2024-11-17T09:40:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:42,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1810 sec 2024-11-17T09:40:42,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:40:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:42,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:42,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:42,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/112f509668a24dacb778a50f7dda54cd is 50, key is test_row_0/A:col10/1731836442011/Put/seqid=0 2024-11-17T09:40:42,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836502167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836502168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836502174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836502177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836502178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741864_1040 (size=12151) 2024-11-17T09:40:42,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836502280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836502290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836502290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836502290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836502291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836502490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836502499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836502500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836502500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836502500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/112f509668a24dacb778a50f7dda54cd 2024-11-17T09:40:42,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/e0d083ba04d4457f972828532ecabd98 is 50, key is test_row_0/B:col10/1731836442011/Put/seqid=0 2024-11-17T09:40:42,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741865_1041 (size=12151) 2024-11-17T09:40:42,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836502800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836502810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836502811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836502811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:42,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836502811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/e0d083ba04d4457f972828532ecabd98 2024-11-17T09:40:43,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a04eaaa769974356baf32385c14db7ee is 50, key is test_row_0/C:col10/1731836442011/Put/seqid=0 2024-11-17T09:40:43,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741866_1042 (size=12151) 2024-11-17T09:40:43,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a04eaaa769974356baf32385c14db7ee 2024-11-17T09:40:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/112f509668a24dacb778a50f7dda54cd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/112f509668a24dacb778a50f7dda54cd 2024-11-17T09:40:43,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/112f509668a24dacb778a50f7dda54cd, entries=150, sequenceid=142, filesize=11.9 K 2024-11-17T09:40:43,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/e0d083ba04d4457f972828532ecabd98 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e0d083ba04d4457f972828532ecabd98 2024-11-17T09:40:43,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e0d083ba04d4457f972828532ecabd98, entries=150, sequenceid=142, filesize=11.9 K 2024-11-17T09:40:43,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a04eaaa769974356baf32385c14db7ee as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a04eaaa769974356baf32385c14db7ee 2024-11-17T09:40:43,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a04eaaa769974356baf32385c14db7ee, entries=150, sequenceid=142, filesize=11.9 K 2024-11-17T09:40:43,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0fd42d9906a7b067f93ee39238302f5 in 934ms, sequenceid=142, compaction requested=true 2024-11-17T09:40:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:43,081 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:43,081 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:43,084 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:43,084 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:43,084 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:43,084 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/12305295f112466f8efea2150279f248, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3e8d4651f35e4e138b437a7ecdde1879, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/deae0acd305b49f4ba09b1a9e78a0a25, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/112f509668a24dacb778a50f7dda54cd] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=47.2 K 2024-11-17T09:40:43,085 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:43,085 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:43,085 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:43,085 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b834a7cbfcb044b8ac5dc958560f7636, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/dac7252d90d244788276cc89b986b4e0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d84b7fbbc1a848f6b4e9003a90710690, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e0d083ba04d4457f972828532ecabd98] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=47.2 K 2024-11-17T09:40:43,086 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12305295f112466f8efea2150279f248, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731836438845 2024-11-17T09:40:43,086 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b834a7cbfcb044b8ac5dc958560f7636, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731836438845 2024-11-17T09:40:43,087 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting dac7252d90d244788276cc89b986b4e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731836440070 2024-11-17T09:40:43,087 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8d4651f35e4e138b437a7ecdde1879, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731836440070 2024-11-17T09:40:43,088 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d84b7fbbc1a848f6b4e9003a90710690, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731836441335 2024-11-17T09:40:43,089 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting deae0acd305b49f4ba09b1a9e78a0a25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731836441335 2024-11-17T09:40:43,089 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 112f509668a24dacb778a50f7dda54cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731836442011 2024-11-17T09:40:43,089 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e0d083ba04d4457f972828532ecabd98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731836442011 2024-11-17T09:40:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T09:40:43,111 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:43,112 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T09:40:43,114 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:43,115 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/2faf1dc8d95243668a11c63a2850d728 is 50, key is test_row_0/B:col10/1731836442011/Put/seqid=0 2024-11-17T09:40:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T09:40:43,115 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:43,117 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:43,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:43,119 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:43,119 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/0b345a979bfb4ee7bd8a34423ff5f830 is 50, key is test_row_0/A:col10/1731836442011/Put/seqid=0 2024-11-17T09:40:43,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741867_1043 (size=12493) 2024-11-17T09:40:43,167 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/2faf1dc8d95243668a11c63a2850d728 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2faf1dc8d95243668a11c63a2850d728 2024-11-17T09:40:43,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741868_1044 (size=12493) 2024-11-17T09:40:43,183 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 2faf1dc8d95243668a11c63a2850d728(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:43,183 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:43,183 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=12, startTime=1731836443081; duration=0sec 2024-11-17T09:40:43,185 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:43,185 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:43,185 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:43,188 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:43,189 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:43,189 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:43,189 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/0afa7bd656ed426789c008934af1234f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6ab16e43861c497bb6507d619e55bc21, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/42a93ee4f0974f428843ee4dd58ddafb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a04eaaa769974356baf32385c14db7ee] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=47.2 K 2024-11-17T09:40:43,190 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0afa7bd656ed426789c008934af1234f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731836438845 2024-11-17T09:40:43,193 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab16e43861c497bb6507d619e55bc21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731836440070 2024-11-17T09:40:43,196 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a93ee4f0974f428843ee4dd58ddafb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731836441335 2024-11-17T09:40:43,197 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a04eaaa769974356baf32385c14db7ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731836442011 2024-11-17T09:40:43,215 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:43,217 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/decd7c7a3b0d4c5e8f299ba25e61ffa7 is 50, key is test_row_0/C:col10/1731836442011/Put/seqid=0 2024-11-17T09:40:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T09:40:43,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741869_1045 (size=12493) 2024-11-17T09:40:43,237 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/decd7c7a3b0d4c5e8f299ba25e61ffa7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/decd7c7a3b0d4c5e8f299ba25e61ffa7 2024-11-17T09:40:43,256 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into decd7c7a3b0d4c5e8f299ba25e61ffa7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:43,256 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:43,256 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=12, startTime=1731836443081; duration=0sec 2024-11-17T09:40:43,257 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:43,258 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:43,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T09:40:43,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:43,270 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T09:40:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:43,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:43,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/9c384d9cd2bc4b08a78d5a38458247de is 50, key is test_row_0/A:col10/1731836442174/Put/seqid=0 2024-11-17T09:40:43,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741870_1046 (size=12151) 2024-11-17T09:40:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:43,313 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/9c384d9cd2bc4b08a78d5a38458247de 2024-11-17T09:40:43,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:43,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/f65e4fb57a6842dbbf82d14304e4c8f9 is 50, key is test_row_0/B:col10/1731836442174/Put/seqid=0 2024-11-17T09:40:43,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741871_1047 (size=12151) 2024-11-17T09:40:43,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836503351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836503352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836503354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836503356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836503358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T09:40:43,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836503470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836503470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836503471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836503480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836503480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,588 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/0b345a979bfb4ee7bd8a34423ff5f830 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0b345a979bfb4ee7bd8a34423ff5f830 2024-11-17T09:40:43,597 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 0b345a979bfb4ee7bd8a34423ff5f830(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:43,597 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:43,597 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=12, startTime=1731836443081; duration=0sec 2024-11-17T09:40:43,598 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:43,598 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:43,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836503680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836503681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836503681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836503690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836503690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T09:40:43,743 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/f65e4fb57a6842dbbf82d14304e4c8f9 2024-11-17T09:40:43,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/f5edb8b975644392a661bfaccc61ded1 is 50, key is test_row_0/C:col10/1731836442174/Put/seqid=0 2024-11-17T09:40:43,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741872_1048 (size=12151) 2024-11-17T09:40:43,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836503990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836503990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836503991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836504000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836504000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,176 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/f5edb8b975644392a661bfaccc61ded1 2024-11-17T09:40:44,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/9c384d9cd2bc4b08a78d5a38458247de as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/9c384d9cd2bc4b08a78d5a38458247de 2024-11-17T09:40:44,193 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/9c384d9cd2bc4b08a78d5a38458247de, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T09:40:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/f65e4fb57a6842dbbf82d14304e4c8f9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/f65e4fb57a6842dbbf82d14304e4c8f9 2024-11-17T09:40:44,209 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/f65e4fb57a6842dbbf82d14304e4c8f9, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T09:40:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/f5edb8b975644392a661bfaccc61ded1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f5edb8b975644392a661bfaccc61ded1 2024-11-17T09:40:44,218 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f5edb8b975644392a661bfaccc61ded1, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T09:40:44,220 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for c0fd42d9906a7b067f93ee39238302f5 in 949ms, sequenceid=158, compaction requested=false 2024-11-17T09:40:44,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:44,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:44,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T09:40:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T09:40:44,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T09:40:44,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1050 sec 2024-11-17T09:40:44,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.1140 sec 2024-11-17T09:40:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T09:40:44,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:44,251 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T09:40:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T09:40:44,255 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:44,257 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:44,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T09:40:44,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T09:40:44,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:44,410 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T09:40:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:44,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:44,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/b30c4c281c1744e8836c1849aea7a8b1 is 50, key is test_row_0/A:col10/1731836443353/Put/seqid=0 2024-11-17T09:40:44,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741873_1049 (size=12151) 2024-11-17T09:40:44,432 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/b30c4c281c1744e8836c1849aea7a8b1 2024-11-17T09:40:44,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/eae2cd0ccb6a49038a7e23c9b725307e is 50, key is test_row_0/B:col10/1731836443353/Put/seqid=0 2024-11-17T09:40:44,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741874_1050 (size=12151) 2024-11-17T09:40:44,467 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/eae2cd0ccb6a49038a7e23c9b725307e 2024-11-17T09:40:44,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5c13b54f13dc4808a57b79f03acec084 is 50, key is test_row_0/C:col10/1731836443353/Put/seqid=0 2024-11-17T09:40:44,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741875_1051 (size=12151) 2024-11-17T09:40:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:44,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:44,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836504518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836504520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836504520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836504523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836504523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T09:40:44,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836504630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836504630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836504630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836504640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836504640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836504840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836504840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836504840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836504849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836504850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:44,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:44,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T09:40:44,904 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5c13b54f13dc4808a57b79f03acec084 2024-11-17T09:40:44,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/b30c4c281c1744e8836c1849aea7a8b1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b30c4c281c1744e8836c1849aea7a8b1 2024-11-17T09:40:44,919 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b30c4c281c1744e8836c1849aea7a8b1, entries=150, sequenceid=178, filesize=11.9 K 2024-11-17T09:40:44,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/eae2cd0ccb6a49038a7e23c9b725307e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/eae2cd0ccb6a49038a7e23c9b725307e 2024-11-17T09:40:44,928 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/eae2cd0ccb6a49038a7e23c9b725307e, entries=150, sequenceid=178, filesize=11.9 K 2024-11-17T09:40:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5c13b54f13dc4808a57b79f03acec084 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5c13b54f13dc4808a57b79f03acec084 2024-11-17T09:40:44,941 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5c13b54f13dc4808a57b79f03acec084, entries=150, sequenceid=178, filesize=11.9 K 2024-11-17T09:40:44,943 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c0fd42d9906a7b067f93ee39238302f5 in 533ms, sequenceid=178, compaction requested=true 2024-11-17T09:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T09:40:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T09:40:44,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T09:40:44,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 689 msec 2024-11-17T09:40:44,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 701 msec 2024-11-17T09:40:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:45,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T09:40:45,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:45,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:45,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:45,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:45,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:45,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:45,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c0120d5a2777465b84cd2cd85bdcc69d is 50, key is test_row_0/A:col10/1731836445152/Put/seqid=0 2024-11-17T09:40:45,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741876_1052 (size=12151) 2024-11-17T09:40:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836505178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836505180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836505189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836505192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836505193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836505300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836505300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836505301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836505301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836505310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T09:40:45,390 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:45,390 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T09:40:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T09:40:45,393 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:45,394 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:45,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:45,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T09:40:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836505510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836505510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836505510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836505510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836505519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T09:40:45,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:45,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:45,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:45,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c0120d5a2777465b84cd2cd85bdcc69d 2024-11-17T09:40:45,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/96fa613796b449cdb6f8eb48f3656ae8 is 50, key is test_row_0/B:col10/1731836445152/Put/seqid=0 2024-11-17T09:40:45,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741877_1053 (size=12151) 2024-11-17T09:40:45,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T09:40:45,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:45,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T09:40:45,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836505820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836505820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836505820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836505821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836505830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:45,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T09:40:45,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:45,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:45,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:45,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:46,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T09:40:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:46,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:46,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/96fa613796b449cdb6f8eb48f3656ae8 2024-11-17T09:40:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T09:40:46,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a4462690140043a79340062c9fb8697b is 50, key is test_row_0/C:col10/1731836445152/Put/seqid=0 2024-11-17T09:40:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741878_1054 (size=12151) 2024-11-17T09:40:46,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a4462690140043a79340062c9fb8697b 2024-11-17T09:40:46,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c0120d5a2777465b84cd2cd85bdcc69d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c0120d5a2777465b84cd2cd85bdcc69d 2024-11-17T09:40:46,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c0120d5a2777465b84cd2cd85bdcc69d, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T09:40:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/96fa613796b449cdb6f8eb48f3656ae8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/96fa613796b449cdb6f8eb48f3656ae8 2024-11-17T09:40:46,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/96fa613796b449cdb6f8eb48f3656ae8, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T09:40:46,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a4462690140043a79340062c9fb8697b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a4462690140043a79340062c9fb8697b 2024-11-17T09:40:46,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a4462690140043a79340062c9fb8697b, entries=150, sequenceid=195, filesize=11.9 K 2024-11-17T09:40:46,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 925ms, sequenceid=195, compaction requested=true 2024-11-17T09:40:46,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:46,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:46,082 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:46,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:46,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:46,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:46,082 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:46,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:46,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:46,084 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:46,084 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:46,084 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:46,084 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:46,084 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:46,084 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2faf1dc8d95243668a11c63a2850d728, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/f65e4fb57a6842dbbf82d14304e4c8f9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/eae2cd0ccb6a49038a7e23c9b725307e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/96fa613796b449cdb6f8eb48f3656ae8] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=47.8 K 2024-11-17T09:40:46,084 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:46,085 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0b345a979bfb4ee7bd8a34423ff5f830, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/9c384d9cd2bc4b08a78d5a38458247de, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b30c4c281c1744e8836c1849aea7a8b1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c0120d5a2777465b84cd2cd85bdcc69d] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=47.8 K 2024-11-17T09:40:46,087 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2faf1dc8d95243668a11c63a2850d728, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731836442011 2024-11-17T09:40:46,087 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b345a979bfb4ee7bd8a34423ff5f830, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731836442011 2024-11-17T09:40:46,087 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c384d9cd2bc4b08a78d5a38458247de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836442154 2024-11-17T09:40:46,087 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting f65e4fb57a6842dbbf82d14304e4c8f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836442154 2024-11-17T09:40:46,089 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting eae2cd0ccb6a49038a7e23c9b725307e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731836443346 2024-11-17T09:40:46,089 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b30c4c281c1744e8836c1849aea7a8b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731836443346 2024-11-17T09:40:46,090 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fa613796b449cdb6f8eb48f3656ae8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731836444515 2024-11-17T09:40:46,090 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0120d5a2777465b84cd2cd85bdcc69d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731836444515 2024-11-17T09:40:46,109 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:46,110 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/e9fc2e969fed4c229a65d6a140f953bf is 50, key is test_row_0/A:col10/1731836445152/Put/seqid=0 2024-11-17T09:40:46,113 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:46,114 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/3088da1e4a244ffa9fb160f2755718f1 is 50, key is test_row_0/B:col10/1731836445152/Put/seqid=0 2024-11-17T09:40:46,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741880_1056 (size=12629) 2024-11-17T09:40:46,133 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/3088da1e4a244ffa9fb160f2755718f1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3088da1e4a244ffa9fb160f2755718f1 2024-11-17T09:40:46,144 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 3088da1e4a244ffa9fb160f2755718f1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:46,144 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:46,144 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=12, startTime=1731836446082; duration=0sec 2024-11-17T09:40:46,144 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:46,144 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:46,145 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:46,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741879_1055 (size=12629) 2024-11-17T09:40:46,147 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:46,147 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:46,147 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:46,147 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/decd7c7a3b0d4c5e8f299ba25e61ffa7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f5edb8b975644392a661bfaccc61ded1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5c13b54f13dc4808a57b79f03acec084, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a4462690140043a79340062c9fb8697b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=47.8 K 2024-11-17T09:40:46,148 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting decd7c7a3b0d4c5e8f299ba25e61ffa7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731836442011 2024-11-17T09:40:46,148 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting f5edb8b975644392a661bfaccc61ded1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836442154 2024-11-17T09:40:46,149 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c13b54f13dc4808a57b79f03acec084, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731836443346 2024-11-17T09:40:46,150 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a4462690140043a79340062c9fb8697b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731836444515 2024-11-17T09:40:46,156 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/e9fc2e969fed4c229a65d6a140f953bf as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e9fc2e969fed4c229a65d6a140f953bf 2024-11-17T09:40:46,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T09:40:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:46,165 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:40:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:46,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:46,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:46,169 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into e9fc2e969fed4c229a65d6a140f953bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:46,169 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:46,169 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=12, startTime=1731836446082; duration=0sec 2024-11-17T09:40:46,170 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:46,170 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:46,172 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:46,173 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/d80277a10d494e66b0325fd916bf6628 is 50, key is test_row_0/C:col10/1731836445152/Put/seqid=0 2024-11-17T09:40:46,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/8d7f0eb1aaac45f9809a7afbd449dd02 is 50, key is test_row_0/A:col10/1731836445175/Put/seqid=0 2024-11-17T09:40:46,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741882_1058 (size=12151) 2024-11-17T09:40:46,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741881_1057 (size=12629) 2024-11-17T09:40:46,196 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/8d7f0eb1aaac45f9809a7afbd449dd02 2024-11-17T09:40:46,210 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/d80277a10d494e66b0325fd916bf6628 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/d80277a10d494e66b0325fd916bf6628 2024-11-17T09:40:46,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/7ae1058b72ca4370ad1eaf9b9fca1b35 is 50, key is test_row_0/B:col10/1731836445175/Put/seqid=0 2024-11-17T09:40:46,219 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into d80277a10d494e66b0325fd916bf6628(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:46,219 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:46,220 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=12, startTime=1731836446082; duration=0sec 2024-11-17T09:40:46,220 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:46,220 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:46,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741883_1059 (size=12151) 2024-11-17T09:40:46,229 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/7ae1058b72ca4370ad1eaf9b9fca1b35 2024-11-17T09:40:46,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b0db3adf463f4460942d45333a0e26a4 is 50, key is test_row_0/C:col10/1731836445175/Put/seqid=0 2024-11-17T09:40:46,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741884_1060 (size=12151) 2024-11-17T09:40:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:46,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:46,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836506351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836506355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836506357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836506355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836506358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836506460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836506470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836506470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836506471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836506471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T09:40:46,647 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b0db3adf463f4460942d45333a0e26a4 2024-11-17T09:40:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/8d7f0eb1aaac45f9809a7afbd449dd02 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/8d7f0eb1aaac45f9809a7afbd449dd02 2024-11-17T09:40:46,661 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/8d7f0eb1aaac45f9809a7afbd449dd02, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T09:40:46,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/7ae1058b72ca4370ad1eaf9b9fca1b35 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/7ae1058b72ca4370ad1eaf9b9fca1b35 2024-11-17T09:40:46,669 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/7ae1058b72ca4370ad1eaf9b9fca1b35, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T09:40:46,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b0db3adf463f4460942d45333a0e26a4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b0db3adf463f4460942d45333a0e26a4 2024-11-17T09:40:46,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836506669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,677 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b0db3adf463f4460942d45333a0e26a4, entries=150, sequenceid=212, filesize=11.9 K 2024-11-17T09:40:46,678 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 513ms, sequenceid=212, compaction requested=false 2024-11-17T09:40:46,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:46,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T09:40:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T09:40:46,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T09:40:46,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2870 sec 2024-11-17T09:40:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:46,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:40:46,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2990 sec 2024-11-17T09:40:46,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:46,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:46,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:46,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:46,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:46,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:46,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/b1f981979dae4ecf8614cda2bfa7f120 is 50, key is test_row_0/A:col10/1731836446355/Put/seqid=0 2024-11-17T09:40:46,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741885_1061 (size=12151) 2024-11-17T09:40:46,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/b1f981979dae4ecf8614cda2bfa7f120 2024-11-17T09:40:46,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836506712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836506713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836506722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836506724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/fb2441b721ee4f1b9657bf6e8aa8f2f1 is 50, key is test_row_0/B:col10/1731836446355/Put/seqid=0 2024-11-17T09:40:46,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741886_1062 (size=12151) 2024-11-17T09:40:46,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/fb2441b721ee4f1b9657bf6e8aa8f2f1 2024-11-17T09:40:46,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/3f30ba861afb4ccda3f8dc9152c86848 is 50, key is test_row_0/C:col10/1731836446355/Put/seqid=0 2024-11-17T09:40:46,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741887_1063 (size=12151) 2024-11-17T09:40:46,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836506830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836506830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836506830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836506840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:46,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836506980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836507040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836507040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836507040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836507050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/3f30ba861afb4ccda3f8dc9152c86848 2024-11-17T09:40:47,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/b1f981979dae4ecf8614cda2bfa7f120 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b1f981979dae4ecf8614cda2bfa7f120 2024-11-17T09:40:47,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b1f981979dae4ecf8614cda2bfa7f120, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T09:40:47,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/fb2441b721ee4f1b9657bf6e8aa8f2f1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fb2441b721ee4f1b9657bf6e8aa8f2f1 2024-11-17T09:40:47,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fb2441b721ee4f1b9657bf6e8aa8f2f1, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T09:40:47,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/3f30ba861afb4ccda3f8dc9152c86848 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/3f30ba861afb4ccda3f8dc9152c86848 2024-11-17T09:40:47,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/3f30ba861afb4ccda3f8dc9152c86848, entries=150, sequenceid=233, filesize=11.9 K 2024-11-17T09:40:47,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c0fd42d9906a7b067f93ee39238302f5 in 511ms, sequenceid=233, compaction requested=true 2024-11-17T09:40:47,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:47,200 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:47,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:47,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:47,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:47,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:47,201 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:47,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:47,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:47,202 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:47,202 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:47,202 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:47,202 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e9fc2e969fed4c229a65d6a140f953bf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/8d7f0eb1aaac45f9809a7afbd449dd02, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b1f981979dae4ecf8614cda2bfa7f120] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.1 K 2024-11-17T09:40:47,203 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:47,203 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9fc2e969fed4c229a65d6a140f953bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731836444515 2024-11-17T09:40:47,203 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:47,203 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:47,203 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3088da1e4a244ffa9fb160f2755718f1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/7ae1058b72ca4370ad1eaf9b9fca1b35, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fb2441b721ee4f1b9657bf6e8aa8f2f1] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.1 K 2024-11-17T09:40:47,203 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d7f0eb1aaac45f9809a7afbd449dd02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731836445175 2024-11-17T09:40:47,204 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3088da1e4a244ffa9fb160f2755718f1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731836444515 2024-11-17T09:40:47,204 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae1058b72ca4370ad1eaf9b9fca1b35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731836445175 2024-11-17T09:40:47,204 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f981979dae4ecf8614cda2bfa7f120, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836446351 2024-11-17T09:40:47,206 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2441b721ee4f1b9657bf6e8aa8f2f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836446351 2024-11-17T09:40:47,218 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:47,218 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:47,219 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/09c974307aca43eeb886ece777a8213e is 50, key is test_row_0/A:col10/1731836446355/Put/seqid=0 2024-11-17T09:40:47,219 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/21400e8361924df0b6a23cf558430f59 is 50, key is test_row_0/B:col10/1731836446355/Put/seqid=0 2024-11-17T09:40:47,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741888_1064 (size=12731) 2024-11-17T09:40:47,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741889_1065 (size=12731) 2024-11-17T09:40:47,253 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/09c974307aca43eeb886ece777a8213e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/09c974307aca43eeb886ece777a8213e 2024-11-17T09:40:47,268 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 09c974307aca43eeb886ece777a8213e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:47,268 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:47,268 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836447200; duration=0sec 2024-11-17T09:40:47,268 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:47,268 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:47,268 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:47,272 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:47,272 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:47,272 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:47,272 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/d80277a10d494e66b0325fd916bf6628, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b0db3adf463f4460942d45333a0e26a4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/3f30ba861afb4ccda3f8dc9152c86848] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.1 K 2024-11-17T09:40:47,273 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d80277a10d494e66b0325fd916bf6628, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731836444515 2024-11-17T09:40:47,273 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0db3adf463f4460942d45333a0e26a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731836445175 2024-11-17T09:40:47,275 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f30ba861afb4ccda3f8dc9152c86848, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836446351 2024-11-17T09:40:47,298 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#53 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:47,299 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6d0a5793c98d4a17ba7c39b742e5702e is 50, key is test_row_0/C:col10/1731836446355/Put/seqid=0 2024-11-17T09:40:47,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741890_1066 (size=12731) 2024-11-17T09:40:47,322 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6d0a5793c98d4a17ba7c39b742e5702e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6d0a5793c98d4a17ba7c39b742e5702e 2024-11-17T09:40:47,330 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into 6d0a5793c98d4a17ba7c39b742e5702e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:47,330 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:47,330 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836447201; duration=0sec 2024-11-17T09:40:47,330 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:47,330 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:47,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T09:40:47,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:47,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:47,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:47,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:47,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:47,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:47,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c7c5f8c6f07b4065bf5de12dfc08e4af is 50, key is test_row_0/A:col10/1731836446718/Put/seqid=0 2024-11-17T09:40:47,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741891_1067 (size=14541) 2024-11-17T09:40:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836507386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836507387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836507390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836507392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836507490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836507499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836507500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836507510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836507510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T09:40:47,540 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:47,540 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T09:40:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T09:40:47,543 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:47,544 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:47,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:47,639 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/21400e8361924df0b6a23cf558430f59 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/21400e8361924df0b6a23cf558430f59 2024-11-17T09:40:47,646 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 21400e8361924df0b6a23cf558430f59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:47,646 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:47,646 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836447201; duration=0sec 2024-11-17T09:40:47,646 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:47,646 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T09:40:47,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T09:40:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:47,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:47,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:47,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836507710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836507710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836507720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836507720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:47,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c7c5f8c6f07b4065bf5de12dfc08e4af 2024-11-17T09:40:47,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/a2bdd58f053b4336b271f817c144e4f8 is 50, key is test_row_0/B:col10/1731836446718/Put/seqid=0 2024-11-17T09:40:47,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741892_1068 (size=12151) 2024-11-17T09:40:47,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/a2bdd58f053b4336b271f817c144e4f8 2024-11-17T09:40:47,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5349fa1c1590441fab54660ac7eca784 is 50, key is test_row_0/C:col10/1731836446718/Put/seqid=0 2024-11-17T09:40:47,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741893_1069 (size=12151) 2024-11-17T09:40:47,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5349fa1c1590441fab54660ac7eca784 2024-11-17T09:40:47,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c7c5f8c6f07b4065bf5de12dfc08e4af as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c7c5f8c6f07b4065bf5de12dfc08e4af 2024-11-17T09:40:47,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c7c5f8c6f07b4065bf5de12dfc08e4af, entries=200, sequenceid=250, filesize=14.2 K 2024-11-17T09:40:47,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/a2bdd58f053b4336b271f817c144e4f8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/a2bdd58f053b4336b271f817c144e4f8 2024-11-17T09:40:47,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/a2bdd58f053b4336b271f817c144e4f8, entries=150, sequenceid=250, filesize=11.9 K 2024-11-17T09:40:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5349fa1c1590441fab54660ac7eca784 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5349fa1c1590441fab54660ac7eca784 2024-11-17T09:40:47,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5349fa1c1590441fab54660ac7eca784, entries=150, sequenceid=250, filesize=11.9 K 2024-11-17T09:40:47,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c0fd42d9906a7b067f93ee39238302f5 in 490ms, sequenceid=250, compaction requested=false 2024-11-17T09:40:47,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:47,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T09:40:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:47,851 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:40:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T09:40:47,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/1c05fc8c0023495f86c8dc922f0d17a7 is 50, key is test_row_0/A:col10/1731836447390/Put/seqid=0 2024-11-17T09:40:47,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741894_1070 (size=12301) 2024-11-17T09:40:47,877 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/1c05fc8c0023495f86c8dc922f0d17a7 2024-11-17T09:40:47,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/e183bff307674dd3a465c5e1ae627bda is 50, key is test_row_0/B:col10/1731836447390/Put/seqid=0 2024-11-17T09:40:47,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741895_1071 (size=12301) 2024-11-17T09:40:47,894 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/e183bff307674dd3a465c5e1ae627bda 2024-11-17T09:40:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/ed6bf8b7ff0d4bd59b8b32dc4016f261 is 50, key is test_row_0/C:col10/1731836447390/Put/seqid=0 2024-11-17T09:40:47,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741896_1072 (size=12301) 2024-11-17T09:40:47,918 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/ed6bf8b7ff0d4bd59b8b32dc4016f261 2024-11-17T09:40:47,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/1c05fc8c0023495f86c8dc922f0d17a7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/1c05fc8c0023495f86c8dc922f0d17a7 2024-11-17T09:40:47,935 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/1c05fc8c0023495f86c8dc922f0d17a7, entries=150, sequenceid=268, filesize=12.0 K 2024-11-17T09:40:47,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/e183bff307674dd3a465c5e1ae627bda as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e183bff307674dd3a465c5e1ae627bda 2024-11-17T09:40:47,943 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e183bff307674dd3a465c5e1ae627bda, entries=150, sequenceid=268, filesize=12.0 K 2024-11-17T09:40:47,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/ed6bf8b7ff0d4bd59b8b32dc4016f261 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ed6bf8b7ff0d4bd59b8b32dc4016f261 2024-11-17T09:40:47,950 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ed6bf8b7ff0d4bd59b8b32dc4016f261, entries=150, sequenceid=268, filesize=12.0 K 2024-11-17T09:40:47,951 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for c0fd42d9906a7b067f93ee39238302f5 in 100ms, sequenceid=268, compaction requested=true 2024-11-17T09:40:47,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:47,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:47,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T09:40:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T09:40:47,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T09:40:47,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 409 msec 2024-11-17T09:40:47,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 417 msec 2024-11-17T09:40:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:48,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:40:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:48,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ba44bae9f6064a0a9ab2fe7435ff0341 is 50, key is test_row_0/A:col10/1731836448035/Put/seqid=0 2024-11-17T09:40:48,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741897_1073 (size=14741) 2024-11-17T09:40:48,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ba44bae9f6064a0a9ab2fe7435ff0341 2024-11-17T09:40:48,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/6dad8d72dcf94e709d521d0b9020c828 is 50, key is test_row_0/B:col10/1731836448035/Put/seqid=0 2024-11-17T09:40:48,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741898_1074 (size=12301) 2024-11-17T09:40:48,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/6dad8d72dcf94e709d521d0b9020c828 2024-11-17T09:40:48,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836448080, max timestamp=1731836448096Num uniques 859; 2024-11-17T09:40:48,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:40:48,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836448080, max timestamp=1731836448103Num uniques 859; 2024-11-17T09:40:48,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:40:48,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731836448080, max timestamp=1731836448096Num uniques 859; 2024-11-17T09:40:48,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-17T09:40:48,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/4c1536ed408449248a0376126686b22e is 50, key is test_row_0/C:col10/1731836448035/Put/seqid=0 2024-11-17T09:40:48,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741899_1075 (size=12301) 2024-11-17T09:40:48,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/4c1536ed408449248a0376126686b22e 2024-11-17T09:40:48,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ba44bae9f6064a0a9ab2fe7435ff0341 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ba44bae9f6064a0a9ab2fe7435ff0341 2024-11-17T09:40:48,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ba44bae9f6064a0a9ab2fe7435ff0341, entries=200, sequenceid=278, filesize=14.4 K 2024-11-17T09:40:48,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/6dad8d72dcf94e709d521d0b9020c828 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/6dad8d72dcf94e709d521d0b9020c828 2024-11-17T09:40:48,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/6dad8d72dcf94e709d521d0b9020c828, entries=150, sequenceid=278, filesize=12.0 K 2024-11-17T09:40:48,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/4c1536ed408449248a0376126686b22e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/4c1536ed408449248a0376126686b22e 2024-11-17T09:40:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836508144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/4c1536ed408449248a0376126686b22e, entries=150, sequenceid=278, filesize=12.0 K 2024-11-17T09:40:48,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.55 KB/134705 for c0fd42d9906a7b067f93ee39238302f5 in 121ms, sequenceid=278, compaction requested=true 2024-11-17T09:40:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:48,160 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:48,160 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:48,162 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54314 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:48,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:48,162 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:48,162 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,163 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/09c974307aca43eeb886ece777a8213e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c7c5f8c6f07b4065bf5de12dfc08e4af, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/1c05fc8c0023495f86c8dc922f0d17a7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ba44bae9f6064a0a9ab2fe7435ff0341] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=53.0 K 2024-11-17T09:40:48,164 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:48,164 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:48,164 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,164 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/21400e8361924df0b6a23cf558430f59, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/a2bdd58f053b4336b271f817c144e4f8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e183bff307674dd3a465c5e1ae627bda, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/6dad8d72dcf94e709d521d0b9020c828] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=48.3 K 2024-11-17T09:40:48,164 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09c974307aca43eeb886ece777a8213e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836446351 2024-11-17T09:40:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:48,166 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 21400e8361924df0b6a23cf558430f59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836446351 2024-11-17T09:40:48,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=138.26 KB heapSize=394.70 KB 2024-11-17T09:40:48,167 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7c5f8c6f07b4065bf5de12dfc08e4af, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731836446705 2024-11-17T09:40:48,167 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a2bdd58f053b4336b271f817c144e4f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731836446705 2024-11-17T09:40:48,167 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c05fc8c0023495f86c8dc922f0d17a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731836447383 2024-11-17T09:40:48,167 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e183bff307674dd3a465c5e1ae627bda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731836447383 2024-11-17T09:40:48,168 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba44bae9f6064a0a9ab2fe7435ff0341, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731836448027 2024-11-17T09:40:48,168 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dad8d72dcf94e709d521d0b9020c828, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731836448027 2024-11-17T09:40:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T09:40:48,170 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:48,171 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T09:40:48,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836448080, max timestamp=1731836448164Num uniques 1018; 2024-11-17T09:40:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T09:40:48,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T09:40:48,173 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:48,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836448080, max timestamp=1731836448164Num uniques 1009; 2024-11-17T09:40:48,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:48,174 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:48,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-17T09:40:48,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:48,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T09:40:48,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:48,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:48,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:48,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:48,188 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#63 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:48,189 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/4510bac6da254bbfa009e318c1d407c3 is 50, key is test_row_0/B:col10/1731836448035/Put/seqid=0 2024-11-17T09:40:48,195 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:48,196 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/17033a6c7314472dbcbb604d49fbfaf8 is 50, key is test_row_0/A:col10/1731836448035/Put/seqid=0 2024-11-17T09:40:48,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/a629078767f642c5848b47a736710cb6 is 50, key is test_row_0/A:col10/1731836448153/Put/seqid=0 2024-11-17T09:40:48,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741900_1076 (size=13017) 2024-11-17T09:40:48,220 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/4510bac6da254bbfa009e318c1d407c3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4510bac6da254bbfa009e318c1d407c3 2024-11-17T09:40:48,230 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 4510bac6da254bbfa009e318c1d407c3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:48,230 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:48,231 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=12, startTime=1731836448160; duration=0sec 2024-11-17T09:40:48,231 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:48,232 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:48,232 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:48,234 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:48,234 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:48,234 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,234 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6d0a5793c98d4a17ba7c39b742e5702e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5349fa1c1590441fab54660ac7eca784, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ed6bf8b7ff0d4bd59b8b32dc4016f261, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/4c1536ed408449248a0376126686b22e] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=48.3 K 2024-11-17T09:40:48,235 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0a5793c98d4a17ba7c39b742e5702e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836446351 2024-11-17T09:40:48,235 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5349fa1c1590441fab54660ac7eca784, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731836446705 2024-11-17T09:40:48,236 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ed6bf8b7ff0d4bd59b8b32dc4016f261, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731836447383 2024-11-17T09:40:48,236 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1536ed408449248a0376126686b22e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731836448027 2024-11-17T09:40:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836508232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741901_1077 (size=13017) 2024-11-17T09:40:48,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836508241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836508243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,256 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/17033a6c7314472dbcbb604d49fbfaf8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/17033a6c7314472dbcbb604d49fbfaf8 2024-11-17T09:40:48,264 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 17033a6c7314472dbcbb604d49fbfaf8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:48,264 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:48,264 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=12, startTime=1731836448160; duration=0sec 2024-11-17T09:40:48,264 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:48,265 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:48,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836508270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741902_1078 (size=17181) 2024-11-17T09:40:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T09:40:48,285 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#66 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:48,289 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a23e2fa7f72247c5b261bb0e36538a6a is 50, key is test_row_0/C:col10/1731836448035/Put/seqid=0 2024-11-17T09:40:48,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741903_1079 (size=13017) 2024-11-17T09:40:48,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:48,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:48,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836508351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836508360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836508360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836508480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:48,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T09:40:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836508510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,512 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836508569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836508570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836508570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:48,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:48,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/a629078767f642c5848b47a736710cb6 2024-11-17T09:40:48,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/9d0a4852b02f48aaa80ddc01229dfe71 is 50, key is test_row_0/B:col10/1731836448153/Put/seqid=0 2024-11-17T09:40:48,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741904_1080 (size=12301) 2024-11-17T09:40:48,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/9d0a4852b02f48aaa80ddc01229dfe71 2024-11-17T09:40:48,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/7f8799e6e63e49cfa7d3bd066c64d5da is 50, key is test_row_0/C:col10/1731836448153/Put/seqid=0 2024-11-17T09:40:48,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741905_1081 (size=12301) 2024-11-17T09:40:48,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/7f8799e6e63e49cfa7d3bd066c64d5da 2024-11-17T09:40:48,726 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a23e2fa7f72247c5b261bb0e36538a6a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a23e2fa7f72247c5b261bb0e36538a6a 2024-11-17T09:40:48,734 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into a23e2fa7f72247c5b261bb0e36538a6a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:48,734 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:48,734 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=12, startTime=1731836448162; duration=0sec 2024-11-17T09:40:48,734 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:48,734 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:48,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/a629078767f642c5848b47a736710cb6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a629078767f642c5848b47a736710cb6 2024-11-17T09:40:48,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a629078767f642c5848b47a736710cb6, entries=250, sequenceid=316, filesize=16.8 K 2024-11-17T09:40:48,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/9d0a4852b02f48aaa80ddc01229dfe71 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/9d0a4852b02f48aaa80ddc01229dfe71 2024-11-17T09:40:48,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/9d0a4852b02f48aaa80ddc01229dfe71, entries=150, sequenceid=316, filesize=12.0 K 2024-11-17T09:40:48,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/7f8799e6e63e49cfa7d3bd066c64d5da as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/7f8799e6e63e49cfa7d3bd066c64d5da 2024-11-17T09:40:48,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/7f8799e6e63e49cfa7d3bd066c64d5da, entries=150, sequenceid=316, filesize=12.0 K 2024-11-17T09:40:48,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~199.06 KB/203840, currentSize=107.34 KB/109920 for c0fd42d9906a7b067f93ee39238302f5 in 605ms, sequenceid=316, compaction requested=false 2024-11-17T09:40:48,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:48,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:40:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:48,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:48,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:48,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:48,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T09:40:48,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/a9b01c26ae794ba2a933e078d2a7d5d9 is 50, key is test_row_0/A:col10/1731836448228/Put/seqid=0 2024-11-17T09:40:48,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836508833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741906_1082 (size=12301) 2024-11-17T09:40:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/a9b01c26ae794ba2a933e078d2a7d5d9 2024-11-17T09:40:48,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/5e801b26a0a6462eaee966d3e0711042 is 50, key is test_row_0/B:col10/1731836448228/Put/seqid=0 2024-11-17T09:40:48,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741907_1083 (size=12301) 2024-11-17T09:40:48,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836508880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836508880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836508880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:48,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:48,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836508950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:48,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836509159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:49,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:49,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/5e801b26a0a6462eaee966d3e0711042 2024-11-17T09:40:49,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/dcf2f5f04d7b486b8e839ee8486d5e16 is 50, key is test_row_0/C:col10/1731836448228/Put/seqid=0 2024-11-17T09:40:49,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741908_1084 (size=12301) 2024-11-17T09:40:49,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/dcf2f5f04d7b486b8e839ee8486d5e16 2024-11-17T09:40:49,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/a9b01c26ae794ba2a933e078d2a7d5d9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a9b01c26ae794ba2a933e078d2a7d5d9 2024-11-17T09:40:49,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a9b01c26ae794ba2a933e078d2a7d5d9, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T09:40:49,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/5e801b26a0a6462eaee966d3e0711042 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/5e801b26a0a6462eaee966d3e0711042 2024-11-17T09:40:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T09:40:49,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/5e801b26a0a6462eaee966d3e0711042, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T09:40:49,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/dcf2f5f04d7b486b8e839ee8486d5e16 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dcf2f5f04d7b486b8e839ee8486d5e16 2024-11-17T09:40:49,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dcf2f5f04d7b486b8e839ee8486d5e16, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T09:40:49,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c0fd42d9906a7b067f93ee39238302f5 in 530ms, sequenceid=339, compaction requested=true 2024-11-17T09:40:49,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:49,324 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:49,324 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:49,325 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:49,325 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:49,325 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:49,325 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:49,325 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,325 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,325 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4510bac6da254bbfa009e318c1d407c3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/9d0a4852b02f48aaa80ddc01229dfe71, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/5e801b26a0a6462eaee966d3e0711042] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.7 K 2024-11-17T09:40:49,325 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/17033a6c7314472dbcbb604d49fbfaf8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a629078767f642c5848b47a736710cb6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a9b01c26ae794ba2a933e078d2a7d5d9] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=41.5 K 2024-11-17T09:40:49,326 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4510bac6da254bbfa009e318c1d407c3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731836448027 2024-11-17T09:40:49,326 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17033a6c7314472dbcbb604d49fbfaf8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731836448027 2024-11-17T09:40:49,327 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0a4852b02f48aaa80ddc01229dfe71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731836448150 2024-11-17T09:40:49,327 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a629078767f642c5848b47a736710cb6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731836448080 2024-11-17T09:40:49,327 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e801b26a0a6462eaee966d3e0711042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836448216 2024-11-17T09:40:49,327 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b01c26ae794ba2a933e078d2a7d5d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836448216 2024-11-17T09:40:49,347 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#72 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:49,348 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:49,348 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d70afd3edb124d17a4b28438d441f0fe is 50, key is test_row_0/B:col10/1731836448228/Put/seqid=0 2024-11-17T09:40:49,348 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/835a0c01bf05422194521ada73e87b2d is 50, key is test_row_0/A:col10/1731836448228/Put/seqid=0 2024-11-17T09:40:49,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741910_1086 (size=13119) 2024-11-17T09:40:49,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741909_1085 (size=13119) 2024-11-17T09:40:49,371 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d70afd3edb124d17a4b28438d441f0fe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d70afd3edb124d17a4b28438d441f0fe 2024-11-17T09:40:49,380 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into d70afd3edb124d17a4b28438d441f0fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:49,380 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:49,380 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836449324; duration=0sec 2024-11-17T09:40:49,381 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:49,381 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:49,381 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:49,382 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:49,382 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:49,382 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,382 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a23e2fa7f72247c5b261bb0e36538a6a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/7f8799e6e63e49cfa7d3bd066c64d5da, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dcf2f5f04d7b486b8e839ee8486d5e16] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.7 K 2024-11-17T09:40:49,383 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a23e2fa7f72247c5b261bb0e36538a6a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731836448027 2024-11-17T09:40:49,384 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8799e6e63e49cfa7d3bd066c64d5da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731836448150 2024-11-17T09:40:49,384 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf2f5f04d7b486b8e839ee8486d5e16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836448216 2024-11-17T09:40:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:49,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T09:40:49,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:49,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:49,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:49,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:49,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:49,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:49,397 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:49,398 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5ac83d01f7934a1dab4aed51d3b1ac08 is 50, key is test_row_0/C:col10/1731836448228/Put/seqid=0 2024-11-17T09:40:49,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/4418060699564e7493ac659677a918e1 is 50, key is test_row_0/A:col10/1731836449392/Put/seqid=0 2024-11-17T09:40:49,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:49,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:49,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741911_1087 (size=13119) 2024-11-17T09:40:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,424 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5ac83d01f7934a1dab4aed51d3b1ac08 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5ac83d01f7934a1dab4aed51d3b1ac08 2024-11-17T09:40:49,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741912_1088 (size=12301) 2024-11-17T09:40:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836509427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,434 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into 5ac83d01f7934a1dab4aed51d3b1ac08(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836509430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,434 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,434 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836449324; duration=0sec 2024-11-17T09:40:49,434 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,434 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/4418060699564e7493ac659677a918e1 2024-11-17T09:40:49,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836509433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/1f49e40d778840ffb7487ee8888fa94e is 50, key is test_row_0/B:col10/1731836449392/Put/seqid=0 2024-11-17T09:40:49,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741913_1089 (size=12301) 2024-11-17T09:40:49,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836509469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836509540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836509540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836509549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:49,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:49,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:49,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836509749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836509759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836509760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:49,773 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/835a0c01bf05422194521ada73e87b2d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/835a0c01bf05422194521ada73e87b2d 2024-11-17T09:40:49,780 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 835a0c01bf05422194521ada73e87b2d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:49,780 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:49,780 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836449323; duration=0sec 2024-11-17T09:40:49,780 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:49,780 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:49,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/1f49e40d778840ffb7487ee8888fa94e 2024-11-17T09:40:49,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5472af6a424a4d25aaf60bbf371425fe is 50, key is test_row_0/C:col10/1731836449392/Put/seqid=0 2024-11-17T09:40:49,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741914_1090 (size=12301) 2024-11-17T09:40:49,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5472af6a424a4d25aaf60bbf371425fe 2024-11-17T09:40:49,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:49,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/4418060699564e7493ac659677a918e1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/4418060699564e7493ac659677a918e1 2024-11-17T09:40:49,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/4418060699564e7493ac659677a918e1, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T09:40:49,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/1f49e40d778840ffb7487ee8888fa94e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/1f49e40d778840ffb7487ee8888fa94e 2024-11-17T09:40:49,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/1f49e40d778840ffb7487ee8888fa94e, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T09:40:49,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/5472af6a424a4d25aaf60bbf371425fe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5472af6a424a4d25aaf60bbf371425fe 2024-11-17T09:40:49,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5472af6a424a4d25aaf60bbf371425fe, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T09:40:49,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c0fd42d9906a7b067f93ee39238302f5 in 520ms, sequenceid=354, compaction requested=false 2024-11-17T09:40:49,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:49,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T09:40:49,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:49,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:49,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:49,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:49,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:49,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:49,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/be7c6f96cc1d4fc290decae8ea07079f is 50, key is test_row_0/A:col10/1731836449424/Put/seqid=0 2024-11-17T09:40:50,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741915_1091 (size=14741) 2024-11-17T09:40:50,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836510024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:50,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:50,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836510059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836510069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836510069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836510140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:50,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:50,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T09:40:50,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:50,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836510350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/be7c6f96cc1d4fc290decae8ea07079f 2024-11-17T09:40:50,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/61600f48aa3d4d87b37b79797b800327 is 50, key is test_row_0/B:col10/1731836449424/Put/seqid=0 2024-11-17T09:40:50,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741916_1092 (size=12301) 2024-11-17T09:40:50,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/61600f48aa3d4d87b37b79797b800327 2024-11-17T09:40:50,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/69c739185c5e43289a3e218b150c9fa7 is 50, key is test_row_0/C:col10/1731836449424/Put/seqid=0 2024-11-17T09:40:50,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741917_1093 (size=12301) 2024-11-17T09:40:50,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/69c739185c5e43289a3e218b150c9fa7 2024-11-17T09:40:50,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/be7c6f96cc1d4fc290decae8ea07079f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/be7c6f96cc1d4fc290decae8ea07079f 2024-11-17T09:40:50,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/be7c6f96cc1d4fc290decae8ea07079f, entries=200, sequenceid=375, filesize=14.4 K 2024-11-17T09:40:50,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/61600f48aa3d4d87b37b79797b800327 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/61600f48aa3d4d87b37b79797b800327 2024-11-17T09:40:50,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/61600f48aa3d4d87b37b79797b800327, entries=150, sequenceid=375, filesize=12.0 K 2024-11-17T09:40:50,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/69c739185c5e43289a3e218b150c9fa7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/69c739185c5e43289a3e218b150c9fa7 2024-11-17T09:40:50,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/69c739185c5e43289a3e218b150c9fa7, entries=150, sequenceid=375, filesize=12.0 K 2024-11-17T09:40:50,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c0fd42d9906a7b067f93ee39238302f5 in 498ms, sequenceid=375, compaction requested=true 2024-11-17T09:40:50,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:50,481 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:50,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:40:50,481 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:50,482 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:50,482 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:50,482 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:50,482 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:50,482 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,482 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,482 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/835a0c01bf05422194521ada73e87b2d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/4418060699564e7493ac659677a918e1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/be7c6f96cc1d4fc290decae8ea07079f] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=39.2 K 2024-11-17T09:40:50,482 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d70afd3edb124d17a4b28438d441f0fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/1f49e40d778840ffb7487ee8888fa94e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/61600f48aa3d4d87b37b79797b800327] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.8 K 2024-11-17T09:40:50,483 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d70afd3edb124d17a4b28438d441f0fe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836448216 2024-11-17T09:40:50,483 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835a0c01bf05422194521ada73e87b2d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836448216 2024-11-17T09:40:50,483 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f49e40d778840ffb7487ee8888fa94e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731836448821 2024-11-17T09:40:50,483 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4418060699564e7493ac659677a918e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731836448821 2024-11-17T09:40:50,484 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 61600f48aa3d4d87b37b79797b800327, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731836449424 2024-11-17T09:40:50,484 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7c6f96cc1d4fc290decae8ea07079f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731836449424 2024-11-17T09:40:50,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T09:40:50,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,498 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T09:40:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:50,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:50,502 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:50,503 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d4c1e80a6f434225b2d21e35b95fb9f4 is 50, key is test_row_0/B:col10/1731836449424/Put/seqid=0 2024-11-17T09:40:50,504 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:50,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ecc83add2c974059be438eccaed7c201 is 50, key is test_row_0/A:col10/1731836450014/Put/seqid=0 2024-11-17T09:40:50,507 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c74f32d867924c70920d27a27f2f0096 is 50, key is test_row_0/A:col10/1731836449424/Put/seqid=0 2024-11-17T09:40:50,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741919_1095 (size=12301) 2024-11-17T09:40:50,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741918_1094 (size=13221) 2024-11-17T09:40:50,522 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ecc83add2c974059be438eccaed7c201 2024-11-17T09:40:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:50,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/22a957bbe0f0472d96f0a93e3fbf668e is 50, key is test_row_0/B:col10/1731836450014/Put/seqid=0 2024-11-17T09:40:50,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741920_1096 (size=13221) 2024-11-17T09:40:50,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741921_1097 (size=12301) 2024-11-17T09:40:50,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836510589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836510603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731836450578, max timestamp=1731836450594Num uniques 859; 2024-11-17T09:40:50,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836450578, max timestamp=1731836450594Num uniques 859; 2024-11-17T09:40:50,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:40:50,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:40:50,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836450578, max timestamp=1731836450594Num uniques 859; 2024-11-17T09:40:50,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836510672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836510682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836510684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836510720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836510739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836510790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836510800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836510800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,929 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/d4c1e80a6f434225b2d21e35b95fb9f4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4c1e80a6f434225b2d21e35b95fb9f4 2024-11-17T09:40:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836510929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,936 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into d4c1e80a6f434225b2d21e35b95fb9f4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:50,936 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:50,936 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836450481; duration=0sec 2024-11-17T09:40:50,936 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:50,937 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:50,937 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:50,938 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:50,938 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:50,938 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:50,938 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5ac83d01f7934a1dab4aed51d3b1ac08, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5472af6a424a4d25aaf60bbf371425fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/69c739185c5e43289a3e218b150c9fa7] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.8 K 2024-11-17T09:40:50,939 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac83d01f7934a1dab4aed51d3b1ac08, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836448216 2024-11-17T09:40:50,939 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5472af6a424a4d25aaf60bbf371425fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731836448821 2024-11-17T09:40:50,940 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c739185c5e43289a3e218b150c9fa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731836449424 2024-11-17T09:40:50,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836510950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:50,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:50,958 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c74f32d867924c70920d27a27f2f0096 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c74f32d867924c70920d27a27f2f0096 2024-11-17T09:40:50,958 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/22a957bbe0f0472d96f0a93e3fbf668e 2024-11-17T09:40:50,971 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into c74f32d867924c70920d27a27f2f0096(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:50,972 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:50,972 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836450480; duration=0sec 2024-11-17T09:40:50,972 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:50,972 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:50,976 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#85 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:50,977 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a01729ca152c4d3b9d153a2017900a1d is 50, key is test_row_0/C:col10/1731836449424/Put/seqid=0 2024-11-17T09:40:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b1ba14b5cc134042a79d2fbd174ceca0 is 50, key is test_row_0/C:col10/1731836450014/Put/seqid=0 2024-11-17T09:40:50,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741922_1098 (size=13221) 2024-11-17T09:40:50,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741923_1099 (size=12301) 2024-11-17T09:40:50,994 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b1ba14b5cc134042a79d2fbd174ceca0 2024-11-17T09:40:51,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ecc83add2c974059be438eccaed7c201 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ecc83add2c974059be438eccaed7c201 2024-11-17T09:40:51,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,005 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ecc83add2c974059be438eccaed7c201, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T09:40:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836510999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/22a957bbe0f0472d96f0a93e3fbf668e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/22a957bbe0f0472d96f0a93e3fbf668e 2024-11-17T09:40:51,012 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/22a957bbe0f0472d96f0a93e3fbf668e, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T09:40:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b1ba14b5cc134042a79d2fbd174ceca0 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b1ba14b5cc134042a79d2fbd174ceca0 2024-11-17T09:40:51,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836511010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836511010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,021 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b1ba14b5cc134042a79d2fbd174ceca0, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T09:40:51,023 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c0fd42d9906a7b067f93ee39238302f5 in 525ms, sequenceid=388, compaction requested=false 2024-11-17T09:40:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T09:40:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T09:40:51,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T09:40:51,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8500 sec 2024-11-17T09:40:51,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.8570 sec 2024-11-17T09:40:51,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-17T09:40:51,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:51,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:51,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:51,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:51,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:51,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:40:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:51,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c8d94abc71db44b3a5483f5bc6de890d is 50, key is test_row_0/A:col10/1731836450677/Put/seqid=0 2024-11-17T09:40:51,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741924_1100 (size=14741) 2024-11-17T09:40:51,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c8d94abc71db44b3a5483f5bc6de890d 2024-11-17T09:40:51,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836511296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836511299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836511311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/fccbe296536840228329b1da0396129d is 50, key is test_row_0/B:col10/1731836450677/Put/seqid=0 2024-11-17T09:40:51,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836511320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836511320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741925_1101 (size=12301) 2024-11-17T09:40:51,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/fccbe296536840228329b1da0396129d 2024-11-17T09:40:51,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/fda25d00040b4208834581c05e33c174 is 50, key is test_row_0/C:col10/1731836450677/Put/seqid=0 2024-11-17T09:40:51,405 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/a01729ca152c4d3b9d153a2017900a1d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a01729ca152c4d3b9d153a2017900a1d 2024-11-17T09:40:51,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836511410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836511410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,421 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into a01729ca152c4d3b9d153a2017900a1d(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:51,421 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:51,421 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836450481; duration=0sec 2024-11-17T09:40:51,425 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:51,425 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:51,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741926_1102 (size=12301) 2024-11-17T09:40:51,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/fda25d00040b4208834581c05e33c174 2024-11-17T09:40:51,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c8d94abc71db44b3a5483f5bc6de890d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c8d94abc71db44b3a5483f5bc6de890d 2024-11-17T09:40:51,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c8d94abc71db44b3a5483f5bc6de890d, entries=200, sequenceid=424, filesize=14.4 K 2024-11-17T09:40:51,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/fccbe296536840228329b1da0396129d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fccbe296536840228329b1da0396129d 2024-11-17T09:40:51,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fccbe296536840228329b1da0396129d, entries=150, sequenceid=424, filesize=12.0 K 2024-11-17T09:40:51,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/fda25d00040b4208834581c05e33c174 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/fda25d00040b4208834581c05e33c174 2024-11-17T09:40:51,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/fda25d00040b4208834581c05e33c174, entries=150, sequenceid=424, filesize=12.0 K 2024-11-17T09:40:51,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for c0fd42d9906a7b067f93ee39238302f5 in 266ms, sequenceid=424, compaction requested=true 2024-11-17T09:40:51,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:51,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:51,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:51,512 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:51,512 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:51,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:51,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:51,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:51,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:51,513 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:51,513 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:51,513 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:51,513 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:51,514 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c74f32d867924c70920d27a27f2f0096, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ecc83add2c974059be438eccaed7c201, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c8d94abc71db44b3a5483f5bc6de890d] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=39.3 K 2024-11-17T09:40:51,514 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:51,514 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:51,514 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4c1e80a6f434225b2d21e35b95fb9f4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/22a957bbe0f0472d96f0a93e3fbf668e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fccbe296536840228329b1da0396129d] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.9 K 2024-11-17T09:40:51,514 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c1e80a6f434225b2d21e35b95fb9f4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731836449424 2024-11-17T09:40:51,515 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a957bbe0f0472d96f0a93e3fbf668e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731836450010 2024-11-17T09:40:51,515 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c74f32d867924c70920d27a27f2f0096, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731836449424 2024-11-17T09:40:51,516 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting fccbe296536840228329b1da0396129d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731836450677 2024-11-17T09:40:51,516 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc83add2c974059be438eccaed7c201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731836450010 2024-11-17T09:40:51,518 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8d94abc71db44b3a5483f5bc6de890d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731836450667 2024-11-17T09:40:51,533 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:51,534 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/64007493a9f3422a84892f60c59f0b45 is 50, key is test_row_0/B:col10/1731836450677/Put/seqid=0 2024-11-17T09:40:51,542 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:51,543 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/32d19c85198247dfaabaa159303aa5f4 is 50, key is test_row_0/A:col10/1731836450677/Put/seqid=0 2024-11-17T09:40:51,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741927_1103 (size=13323) 2024-11-17T09:40:51,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741928_1104 (size=13323) 2024-11-17T09:40:51,581 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/32d19c85198247dfaabaa159303aa5f4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/32d19c85198247dfaabaa159303aa5f4 2024-11-17T09:40:51,590 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 32d19c85198247dfaabaa159303aa5f4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:51,590 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:51,590 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836451511; duration=0sec 2024-11-17T09:40:51,591 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:51,591 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:51,591 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:51,593 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:51,593 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:51,593 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:51,593 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a01729ca152c4d3b9d153a2017900a1d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b1ba14b5cc134042a79d2fbd174ceca0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/fda25d00040b4208834581c05e33c174] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=36.9 K 2024-11-17T09:40:51,594 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01729ca152c4d3b9d153a2017900a1d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731836449424 2024-11-17T09:40:51,596 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1ba14b5cc134042a79d2fbd174ceca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731836450010 2024-11-17T09:40:51,596 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda25d00040b4208834581c05e33c174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731836450677 2024-11-17T09:40:51,622 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:51,622 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/df4f40664e0840b9a37f8880409dda22 is 50, key is test_row_0/C:col10/1731836450677/Put/seqid=0 2024-11-17T09:40:51,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T09:40:51,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:51,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:51,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:51,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:51,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:51,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:51,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/355aa5a7a68c42eea3985659cae82b2f is 50, key is test_row_0/A:col10/1731836451622/Put/seqid=0 2024-11-17T09:40:51,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836511680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836511684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741929_1105 (size=13323) 2024-11-17T09:40:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,702 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/df4f40664e0840b9a37f8880409dda22 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/df4f40664e0840b9a37f8880409dda22 2024-11-17T09:40:51,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741930_1106 (size=17181) 2024-11-17T09:40:51,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/355aa5a7a68c42eea3985659cae82b2f 2024-11-17T09:40:51,710 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into df4f40664e0840b9a37f8880409dda22(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:51,710 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:51,710 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836451512; duration=0sec 2024-11-17T09:40:51,710 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:51,710 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:51,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/55faecd9aadc40a0a3a3964991b1b3d8 is 50, key is test_row_0/B:col10/1731836451622/Put/seqid=0 2024-11-17T09:40:51,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741931_1107 (size=12301) 2024-11-17T09:40:51,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/55faecd9aadc40a0a3a3964991b1b3d8 2024-11-17T09:40:51,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/78ace889cc4a4f71ba64adff1fc449e6 is 50, key is test_row_0/C:col10/1731836451622/Put/seqid=0 2024-11-17T09:40:51,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741932_1108 (size=12301) 2024-11-17T09:40:51,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/78ace889cc4a4f71ba64adff1fc449e6 2024-11-17T09:40:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836511790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/355aa5a7a68c42eea3985659cae82b2f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/355aa5a7a68c42eea3985659cae82b2f 2024-11-17T09:40:51,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/355aa5a7a68c42eea3985659cae82b2f, entries=250, sequenceid=440, filesize=16.8 K 2024-11-17T09:40:51,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/55faecd9aadc40a0a3a3964991b1b3d8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/55faecd9aadc40a0a3a3964991b1b3d8 2024-11-17T09:40:51,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836511800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/55faecd9aadc40a0a3a3964991b1b3d8, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T09:40:51,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/78ace889cc4a4f71ba64adff1fc449e6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/78ace889cc4a4f71ba64adff1fc449e6 2024-11-17T09:40:51,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/78ace889cc4a4f71ba64adff1fc449e6, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T09:40:51,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c0fd42d9906a7b067f93ee39238302f5 in 199ms, sequenceid=440, compaction requested=false 2024-11-17T09:40:51,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:51,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T09:40:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:51,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/45478d3881e740ac99c3b11b921a5423 is 50, key is test_row_0/A:col10/1731836451652/Put/seqid=0 2024-11-17T09:40:51,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741933_1109 (size=14741) 2024-11-17T09:40:51,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/45478d3881e740ac99c3b11b921a5423 2024-11-17T09:40:51,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836511860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836511852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836511870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/07c6461814eb4953b9a339598c245c39 is 50, key is test_row_0/B:col10/1731836451652/Put/seqid=0 2024-11-17T09:40:51,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741934_1110 (size=12301) 2024-11-17T09:40:51,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836511981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836511981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836511981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:51,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:51,990 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/64007493a9f3422a84892f60c59f0b45 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/64007493a9f3422a84892f60c59f0b45 2024-11-17T09:40:52,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836511999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,011 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 64007493a9f3422a84892f60c59f0b45(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:52,012 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:52,012 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836451512; duration=0sec 2024-11-17T09:40:52,012 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:52,012 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:52,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836512020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836512190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836512191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836512191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836512310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/07c6461814eb4953b9a339598c245c39 2024-11-17T09:40:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T09:40:52,331 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:52,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T09:40:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T09:40:52,341 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:52,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836512340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/dd8b4809be8b49b587b054378cf472b1 is 50, key is test_row_0/C:col10/1731836451652/Put/seqid=0 2024-11-17T09:40:52,352 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:52,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:52,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741935_1111 (size=12301) 2024-11-17T09:40:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T09:40:52,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836512499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836512500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:52,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836512500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T09:40:52,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:52,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:52,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/dd8b4809be8b49b587b054378cf472b1 2024-11-17T09:40:52,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/45478d3881e740ac99c3b11b921a5423 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/45478d3881e740ac99c3b11b921a5423 2024-11-17T09:40:52,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/45478d3881e740ac99c3b11b921a5423, entries=200, sequenceid=460, filesize=14.4 K 2024-11-17T09:40:52,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/07c6461814eb4953b9a339598c245c39 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/07c6461814eb4953b9a339598c245c39 2024-11-17T09:40:52,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836512829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:52,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/07c6461814eb4953b9a339598c245c39, entries=150, sequenceid=460, filesize=12.0 K 2024-11-17T09:40:52,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/dd8b4809be8b49b587b054378cf472b1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dd8b4809be8b49b587b054378cf472b1 2024-11-17T09:40:52,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dd8b4809be8b49b587b054378cf472b1, entries=150, sequenceid=460, filesize=12.0 K 2024-11-17T09:40:52,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c0fd42d9906a7b067f93ee39238302f5 in 1008ms, sequenceid=460, compaction requested=true 2024-11-17T09:40:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:52,841 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:52,841 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:52,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:52,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:52,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:52,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:52,843 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:52,843 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:52,843 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:52,843 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:52,843 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,843 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,843 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/64007493a9f3422a84892f60c59f0b45, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/55faecd9aadc40a0a3a3964991b1b3d8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/07c6461814eb4953b9a339598c245c39] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=37.0 K 2024-11-17T09:40:52,843 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/32d19c85198247dfaabaa159303aa5f4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/355aa5a7a68c42eea3985659cae82b2f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/45478d3881e740ac99c3b11b921a5423] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=44.2 K 2024-11-17T09:40:52,843 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 64007493a9f3422a84892f60c59f0b45, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731836450677 2024-11-17T09:40:52,843 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d19c85198247dfaabaa159303aa5f4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731836450677 2024-11-17T09:40:52,844 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 55faecd9aadc40a0a3a3964991b1b3d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731836451294 2024-11-17T09:40:52,844 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 355aa5a7a68c42eea3985659cae82b2f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731836451285 2024-11-17T09:40:52,844 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c6461814eb4953b9a339598c245c39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731836451652 2024-11-17T09:40:52,845 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45478d3881e740ac99c3b11b921a5423, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731836451652 2024-11-17T09:40:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:52,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T09:40:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:52,857 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#99 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:52,859 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:52,859 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/2a334c3b86134287aa1c52a1c6864401 is 50, key is test_row_0/B:col10/1731836451652/Put/seqid=0 2024-11-17T09:40:52,861 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/895294d1add847f192a1d8cd2ab3852e is 50, key is test_row_0/A:col10/1731836451652/Put/seqid=0 2024-11-17T09:40:52,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/7274ea1ed09f4999bd636c56ebc3e3b1 is 50, key is test_row_0/A:col10/1731836452853/Put/seqid=0 2024-11-17T09:40:52,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741936_1112 (size=13425) 2024-11-17T09:40:52,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741937_1113 (size=13425) 2024-11-17T09:40:52,936 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/2a334c3b86134287aa1c52a1c6864401 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2a334c3b86134287aa1c52a1c6864401 2024-11-17T09:40:52,947 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 2a334c3b86134287aa1c52a1c6864401(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:52,947 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:52,947 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836452841; duration=0sec 2024-11-17T09:40:52,947 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:52,947 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:52,948 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:52,950 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/895294d1add847f192a1d8cd2ab3852e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/895294d1add847f192a1d8cd2ab3852e 2024-11-17T09:40:52,951 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:52,951 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:52,951 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,952 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/df4f40664e0840b9a37f8880409dda22, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/78ace889cc4a4f71ba64adff1fc449e6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dd8b4809be8b49b587b054378cf472b1] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=37.0 K 2024-11-17T09:40:52,953 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting df4f40664e0840b9a37f8880409dda22, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731836450677 2024-11-17T09:40:52,956 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 78ace889cc4a4f71ba64adff1fc449e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731836451294 2024-11-17T09:40:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T09:40:52,962 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting dd8b4809be8b49b587b054378cf472b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731836451652 2024-11-17T09:40:52,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741938_1114 (size=12301) 2024-11-17T09:40:52,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/7274ea1ed09f4999bd636c56ebc3e3b1 2024-11-17T09:40:52,966 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 895294d1add847f192a1d8cd2ab3852e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:52,967 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:52,967 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836452841; duration=0sec 2024-11-17T09:40:52,967 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:52,967 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:52,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:52,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:52,986 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:52,987 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/bc5133c6dc014701aaf80113b36ae50e is 50, key is test_row_0/C:col10/1731836451652/Put/seqid=0 2024-11-17T09:40:53,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/03b7a684c9614517a477672a91c31f90 is 50, key is test_row_0/B:col10/1731836452853/Put/seqid=0 2024-11-17T09:40:53,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836513015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836513017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836513020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741939_1115 (size=13425) 2024-11-17T09:40:53,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836513036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741940_1116 (size=12301) 2024-11-17T09:40:53,061 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/bc5133c6dc014701aaf80113b36ae50e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bc5133c6dc014701aaf80113b36ae50e 2024-11-17T09:40:53,075 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into bc5133c6dc014701aaf80113b36ae50e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:53,075 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:53,075 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836452842; duration=0sec 2024-11-17T09:40:53,076 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:53,076 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:53,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:53,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836513130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836513131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:53,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:53,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:53,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836513340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836513340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:53,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:53,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/03b7a684c9614517a477672a91c31f90 2024-11-17T09:40:53,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/1a1402e1bb894dd4972eb26114fb002b is 50, key is test_row_0/C:col10/1731836452853/Put/seqid=0 2024-11-17T09:40:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T09:40:53,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741941_1117 (size=12301) 2024-11-17T09:40:53,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/1a1402e1bb894dd4972eb26114fb002b 2024-11-17T09:40:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/7274ea1ed09f4999bd636c56ebc3e3b1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/7274ea1ed09f4999bd636c56ebc3e3b1 2024-11-17T09:40:53,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/7274ea1ed09f4999bd636c56ebc3e3b1, entries=150, sequenceid=475, filesize=12.0 K 2024-11-17T09:40:53,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/03b7a684c9614517a477672a91c31f90 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/03b7a684c9614517a477672a91c31f90 2024-11-17T09:40:53,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/03b7a684c9614517a477672a91c31f90, entries=150, sequenceid=475, filesize=12.0 K 2024-11-17T09:40:53,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/1a1402e1bb894dd4972eb26114fb002b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/1a1402e1bb894dd4972eb26114fb002b 2024-11-17T09:40:53,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/1a1402e1bb894dd4972eb26114fb002b, entries=150, sequenceid=475, filesize=12.0 K 2024-11-17T09:40:53,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c0fd42d9906a7b067f93ee39238302f5 in 705ms, sequenceid=475, compaction requested=false 2024-11-17T09:40:53,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:53,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T09:40:53,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:53,598 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T09:40:53,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:53,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:53,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:53,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:53,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:53,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:53,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/aa04a3110eff4a5e9651ec0eb3c1e62e is 50, key is test_row_0/A:col10/1731836452996/Put/seqid=0 2024-11-17T09:40:53,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741942_1118 (size=12301) 2024-11-17T09:40:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:53,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836513680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836513681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836513791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836513791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:53,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:53,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836513839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:53,843 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836514000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836514000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,030 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/aa04a3110eff4a5e9651ec0eb3c1e62e 2024-11-17T09:40:54,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836514039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,042 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/ae4417d56b17460eb88320d62006a091 is 50, key is test_row_0/B:col10/1731836452996/Put/seqid=0 2024-11-17T09:40:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836514050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,059 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741943_1119 (size=12301) 2024-11-17T09:40:54,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836514310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836514310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,479 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/ae4417d56b17460eb88320d62006a091 2024-11-17T09:40:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T09:40:54,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/202950d0af204454bf8c213e61842f13 is 50, key is test_row_0/C:col10/1731836452996/Put/seqid=0 2024-11-17T09:40:54,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741944_1120 (size=12301) 2024-11-17T09:40:54,522 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/202950d0af204454bf8c213e61842f13 2024-11-17T09:40:54,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/aa04a3110eff4a5e9651ec0eb3c1e62e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/aa04a3110eff4a5e9651ec0eb3c1e62e 2024-11-17T09:40:54,537 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/aa04a3110eff4a5e9651ec0eb3c1e62e, entries=150, sequenceid=496, filesize=12.0 K 2024-11-17T09:40:54,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/ae4417d56b17460eb88320d62006a091 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/ae4417d56b17460eb88320d62006a091 2024-11-17T09:40:54,546 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/ae4417d56b17460eb88320d62006a091, entries=150, sequenceid=496, filesize=12.0 K 2024-11-17T09:40:54,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/202950d0af204454bf8c213e61842f13 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/202950d0af204454bf8c213e61842f13 2024-11-17T09:40:54,553 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/202950d0af204454bf8c213e61842f13, entries=150, sequenceid=496, filesize=12.0 K 2024-11-17T09:40:54,554 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0fd42d9906a7b067f93ee39238302f5 in 957ms, sequenceid=496, compaction requested=true 2024-11-17T09:40:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T09:40:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T09:40:54,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T09:40:54,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2040 sec 2024-11-17T09:40:54,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2310 sec 2024-11-17T09:40:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:54,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T09:40:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:54,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:54,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c14467acb70c4e569f7687d7dc2ee6ab is 50, key is test_row_0/A:col10/1731836453679/Put/seqid=0 2024-11-17T09:40:54,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741945_1121 (size=12301) 2024-11-17T09:40:54,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c14467acb70c4e569f7687d7dc2ee6ab 2024-11-17T09:40:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836514868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/8ceef49027e64d2cbf07b9acb9345f1f is 50, key is test_row_0/B:col10/1731836453679/Put/seqid=0 2024-11-17T09:40:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836514870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741946_1122 (size=12301) 2024-11-17T09:40:54,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/8ceef49027e64d2cbf07b9acb9345f1f 2024-11-17T09:40:54,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6aed7c4be6ba4f11b3043aa4ecd0044d is 50, key is test_row_0/C:col10/1731836453679/Put/seqid=0 2024-11-17T09:40:54,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741947_1123 (size=12301) 2024-11-17T09:40:54,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836514980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:54,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836514980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:54,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836515189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836515200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6aed7c4be6ba4f11b3043aa4ecd0044d 2024-11-17T09:40:55,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/c14467acb70c4e569f7687d7dc2ee6ab as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c14467acb70c4e569f7687d7dc2ee6ab 2024-11-17T09:40:55,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c14467acb70c4e569f7687d7dc2ee6ab, entries=150, sequenceid=512, filesize=12.0 K 2024-11-17T09:40:55,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/8ceef49027e64d2cbf07b9acb9345f1f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8ceef49027e64d2cbf07b9acb9345f1f 2024-11-17T09:40:55,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8ceef49027e64d2cbf07b9acb9345f1f, entries=150, sequenceid=512, filesize=12.0 K 2024-11-17T09:40:55,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/6aed7c4be6ba4f11b3043aa4ecd0044d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6aed7c4be6ba4f11b3043aa4ecd0044d 2024-11-17T09:40:55,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6aed7c4be6ba4f11b3043aa4ecd0044d, entries=150, sequenceid=512, filesize=12.0 K 2024-11-17T09:40:55,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 550ms, sequenceid=512, compaction requested=true 2024-11-17T09:40:55,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:55,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:55,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:55,375 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:55,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:55,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:55,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:55,376 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:55,378 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:55,378 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:55,379 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:55,379 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/895294d1add847f192a1d8cd2ab3852e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/7274ea1ed09f4999bd636c56ebc3e3b1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/aa04a3110eff4a5e9651ec0eb3c1e62e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c14467acb70c4e569f7687d7dc2ee6ab] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=49.1 K 2024-11-17T09:40:55,379 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:55,379 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:55,379 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:55,379 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2a334c3b86134287aa1c52a1c6864401, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/03b7a684c9614517a477672a91c31f90, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/ae4417d56b17460eb88320d62006a091, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8ceef49027e64d2cbf07b9acb9345f1f] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=49.1 K 2024-11-17T09:40:55,380 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a334c3b86134287aa1c52a1c6864401, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731836451652 2024-11-17T09:40:55,380 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 895294d1add847f192a1d8cd2ab3852e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731836451652 2024-11-17T09:40:55,383 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b7a684c9614517a477672a91c31f90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731836451843 2024-11-17T09:40:55,383 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7274ea1ed09f4999bd636c56ebc3e3b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731836451843 2024-11-17T09:40:55,383 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ae4417d56b17460eb88320d62006a091, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731836452979 2024-11-17T09:40:55,384 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa04a3110eff4a5e9651ec0eb3c1e62e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731836452979 2024-11-17T09:40:55,384 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ceef49027e64d2cbf07b9acb9345f1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836453679 2024-11-17T09:40:55,384 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c14467acb70c4e569f7687d7dc2ee6ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836453679 2024-11-17T09:40:55,400 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:55,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/23a3dd27564344f1bec7a0fc7bc0c51d is 50, key is test_row_0/A:col10/1731836453679/Put/seqid=0 2024-11-17T09:40:55,411 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:55,412 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/8b6d71700652472da811e2b6e1f735a4 is 50, key is test_row_0/B:col10/1731836453679/Put/seqid=0 2024-11-17T09:40:55,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741948_1124 (size=13561) 2024-11-17T09:40:55,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741949_1125 (size=13561) 2024-11-17T09:40:55,453 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/23a3dd27564344f1bec7a0fc7bc0c51d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/23a3dd27564344f1bec7a0fc7bc0c51d 2024-11-17T09:40:55,462 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/8b6d71700652472da811e2b6e1f735a4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8b6d71700652472da811e2b6e1f735a4 2024-11-17T09:40:55,464 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into 23a3dd27564344f1bec7a0fc7bc0c51d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:55,464 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:55,465 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=12, startTime=1731836455375; duration=0sec 2024-11-17T09:40:55,465 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:55,465 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:55,465 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:40:55,467 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:40:55,467 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:55,467 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:55,467 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bc5133c6dc014701aaf80113b36ae50e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/1a1402e1bb894dd4972eb26114fb002b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/202950d0af204454bf8c213e61842f13, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6aed7c4be6ba4f11b3043aa4ecd0044d] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=49.1 K 2024-11-17T09:40:55,469 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc5133c6dc014701aaf80113b36ae50e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731836451652 2024-11-17T09:40:55,469 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 8b6d71700652472da811e2b6e1f735a4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:55,469 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:55,470 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=12, startTime=1731836455375; duration=0sec 2024-11-17T09:40:55,470 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:55,470 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:55,470 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1402e1bb894dd4972eb26114fb002b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731836451843 2024-11-17T09:40:55,471 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 202950d0af204454bf8c213e61842f13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731836452979 2024-11-17T09:40:55,471 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aed7c4be6ba4f11b3043aa4ecd0044d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836453679 2024-11-17T09:40:55,489 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:55,489 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/bdd22974f7fd4137b397c1d76ea46414 is 50, key is test_row_0/C:col10/1731836453679/Put/seqid=0 2024-11-17T09:40:55,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:55,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:40:55,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:55,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:55,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:55,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:55,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:55,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:55,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/f65dc85226c148d3b53e8e3b91dd67e4 is 50, key is test_row_0/A:col10/1731836455509/Put/seqid=0 2024-11-17T09:40:55,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741950_1126 (size=13561) 2024-11-17T09:40:55,577 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/bdd22974f7fd4137b397c1d76ea46414 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bdd22974f7fd4137b397c1d76ea46414 2024-11-17T09:40:55,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741951_1127 (size=14741) 2024-11-17T09:40:55,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/f65dc85226c148d3b53e8e3b91dd67e4 2024-11-17T09:40:55,588 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into bdd22974f7fd4137b397c1d76ea46414(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:55,588 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:55,588 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=12, startTime=1731836455375; duration=0sec 2024-11-17T09:40:55,588 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:55,589 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:55,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/710c71ff583b4d8aa54aa074a1cb73c8 is 50, key is test_row_0/B:col10/1731836455509/Put/seqid=0 2024-11-17T09:40:55,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836515594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836515593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741952_1128 (size=12301) 2024-11-17T09:40:55,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836515709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836515720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59460 deadline: 1731836515870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,873 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836515920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:55,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836515929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/710c71ff583b4d8aa54aa074a1cb73c8 2024-11-17T09:40:56,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/20edf59f0b784e3b82b04e8e70574500 is 50, key is test_row_0/C:col10/1731836455509/Put/seqid=0 2024-11-17T09:40:56,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59416 deadline: 1731836516070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59444 deadline: 1731836516070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,074 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,075 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in c0fd42d9906a7b067f93ee39238302f5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:40:56,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741953_1129 (size=12301) 2024-11-17T09:40:56,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/20edf59f0b784e3b82b04e8e70574500 2024-11-17T09:40:56,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/f65dc85226c148d3b53e8e3b91dd67e4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/f65dc85226c148d3b53e8e3b91dd67e4 2024-11-17T09:40:56,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/f65dc85226c148d3b53e8e3b91dd67e4, entries=200, sequenceid=531, filesize=14.4 K 2024-11-17T09:40:56,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/710c71ff583b4d8aa54aa074a1cb73c8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/710c71ff583b4d8aa54aa074a1cb73c8 2024-11-17T09:40:56,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/710c71ff583b4d8aa54aa074a1cb73c8, entries=150, sequenceid=531, filesize=12.0 K 2024-11-17T09:40:56,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/20edf59f0b784e3b82b04e8e70574500 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/20edf59f0b784e3b82b04e8e70574500 2024-11-17T09:40:56,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/20edf59f0b784e3b82b04e8e70574500, entries=150, sequenceid=531, filesize=12.0 K 2024-11-17T09:40:56,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0fd42d9906a7b067f93ee39238302f5 in 613ms, sequenceid=531, compaction requested=false 2024-11-17T09:40:56,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:56,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:40:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:56,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:56,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/e6e7a11ff97247d389c2e7efbd8236af is 50, key is test_row_0/A:col10/1731836456230/Put/seqid=0 2024-11-17T09:40:56,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741954_1130 (size=14741) 2024-11-17T09:40:56,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/e6e7a11ff97247d389c2e7efbd8236af 2024-11-17T09:40:56,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/358c995859054af78cb5c8df622a4b1a is 50, key is test_row_0/B:col10/1731836456230/Put/seqid=0 2024-11-17T09:40:56,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836516282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836516282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741955_1131 (size=12301) 2024-11-17T09:40:56,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/358c995859054af78cb5c8df622a4b1a 2024-11-17T09:40:56,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/8050fddf1610405dbe5975cfd4b2d4fb is 50, key is test_row_0/C:col10/1731836456230/Put/seqid=0 2024-11-17T09:40:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741956_1132 (size=12301) 2024-11-17T09:40:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836516400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836516400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T09:40:56,500 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:56,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:40:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T09:40:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T09:40:56,504 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:40:56,505 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:40:56,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:40:56,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59430 deadline: 1731836516610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T09:40:56,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:40:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59410 deadline: 1731836516617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 2024-11-17T09:40:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:40:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:40:56,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T09:40:56,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:56,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:56,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:40:56,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/8050fddf1610405dbe5975cfd4b2d4fb 2024-11-17T09:40:56,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/e6e7a11ff97247d389c2e7efbd8236af as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e6e7a11ff97247d389c2e7efbd8236af 2024-11-17T09:40:56,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e6e7a11ff97247d389c2e7efbd8236af, entries=200, sequenceid=548, filesize=14.4 K 2024-11-17T09:40:56,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/358c995859054af78cb5c8df622a4b1a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/358c995859054af78cb5c8df622a4b1a 2024-11-17T09:40:56,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/358c995859054af78cb5c8df622a4b1a, entries=150, sequenceid=548, filesize=12.0 K 2024-11-17T09:40:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/8050fddf1610405dbe5975cfd4b2d4fb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8050fddf1610405dbe5975cfd4b2d4fb 2024-11-17T09:40:56,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8050fddf1610405dbe5975cfd4b2d4fb, entries=150, sequenceid=548, filesize=12.0 K 2024-11-17T09:40:56,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0fd42d9906a7b067f93ee39238302f5 in 524ms, sequenceid=548, compaction requested=true 2024-11-17T09:40:56,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:40:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:56,756 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:40:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:56,756 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0fd42d9906a7b067f93ee39238302f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:40:56,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:56,760 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:56,760 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/B is initiating minor compaction (all files) 2024-11-17T09:40:56,760 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/B in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:56,761 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8b6d71700652472da811e2b6e1f735a4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/710c71ff583b4d8aa54aa074a1cb73c8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/358c995859054af78cb5c8df622a4b1a] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=37.3 K 2024-11-17T09:40:56,761 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:56,761 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b6d71700652472da811e2b6e1f735a4, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836453679 2024-11-17T09:40:56,761 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/A is initiating minor compaction (all files) 2024-11-17T09:40:56,761 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/A in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:56,761 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/23a3dd27564344f1bec7a0fc7bc0c51d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/f65dc85226c148d3b53e8e3b91dd67e4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e6e7a11ff97247d389c2e7efbd8236af] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=42.0 K 2024-11-17T09:40:56,761 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 710c71ff583b4d8aa54aa074a1cb73c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731836454866 2024-11-17T09:40:56,762 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a3dd27564344f1bec7a0fc7bc0c51d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836453679 2024-11-17T09:40:56,762 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65dc85226c148d3b53e8e3b91dd67e4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731836454843 2024-11-17T09:40:56,763 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e7a11ff97247d389c2e7efbd8236af, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731836455583 2024-11-17T09:40:56,763 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 358c995859054af78cb5c8df622a4b1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731836455583 2024-11-17T09:40:56,768 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T09:40:56,769 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:40:56,769 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T09:40:56,769 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:40:56,772 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T09:40:56,772 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:40:56,775 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,775 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,775 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,775 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:40:56,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:40:56,776 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,776 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:40:56,776 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T09:40:56,777 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:40:56,777 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,777 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:40:56,782 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#A#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:56,783 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ff17d434586c4596b8c254b887ee0e03 is 50, key is test_row_0/A:col10/1731836456230/Put/seqid=0 2024-11-17T09:40:56,789 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:56,790 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/305d3e3f51e24bd4a822399ae97cdb19 is 50, key is test_row_0/B:col10/1731836456230/Put/seqid=0 2024-11-17T09:40:56,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741957_1133 (size=13663) 2024-11-17T09:40:56,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741958_1134 (size=13663) 2024-11-17T09:40:56,807 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/305d3e3f51e24bd4a822399ae97cdb19 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/305d3e3f51e24bd4a822399ae97cdb19 2024-11-17T09:40:56,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T09:40:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:56,814 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:40:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:40:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:40:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:40:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:40:56,819 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/B of c0fd42d9906a7b067f93ee39238302f5 into 305d3e3f51e24bd4a822399ae97cdb19(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:56,820 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:56,820 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/B, priority=13, startTime=1731836456756; duration=0sec 2024-11-17T09:40:56,820 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:40:56,820 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:B 2024-11-17T09:40:56,820 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:40:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T09:40:56,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/3bc2414932b146ca8190d9b92bbcc747 is 50, key is test_row_0/A:col10/1731836456281/Put/seqid=0 2024-11-17T09:40:56,821 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:40:56,821 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): c0fd42d9906a7b067f93ee39238302f5/C is initiating minor compaction (all files) 2024-11-17T09:40:56,821 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0fd42d9906a7b067f93ee39238302f5/C in TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:56,822 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bdd22974f7fd4137b397c1d76ea46414, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/20edf59f0b784e3b82b04e8e70574500, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8050fddf1610405dbe5975cfd4b2d4fb] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp, totalSize=37.3 K 2024-11-17T09:40:56,822 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting bdd22974f7fd4137b397c1d76ea46414, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836453679 2024-11-17T09:40:56,823 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 20edf59f0b784e3b82b04e8e70574500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1731836454866 2024-11-17T09:40:56,823 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8050fddf1610405dbe5975cfd4b2d4fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731836455583 2024-11-17T09:40:56,838 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0fd42d9906a7b067f93ee39238302f5#C#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:40:56,838 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/40f4e54a8a2b46b697ed47d3f8067789 is 50, key is test_row_0/C:col10/1731836456230/Put/seqid=0 2024-11-17T09:40:56,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741959_1135 (size=12301) 2024-11-17T09:40:56,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741960_1136 (size=13663) 2024-11-17T09:40:56,852 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/40f4e54a8a2b46b697ed47d3f8067789 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/40f4e54a8a2b46b697ed47d3f8067789 2024-11-17T09:40:56,857 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/C of c0fd42d9906a7b067f93ee39238302f5 into 40f4e54a8a2b46b697ed47d3f8067789(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:56,858 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:56,858 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/C, priority=13, startTime=1731836456756; duration=0sec 2024-11-17T09:40:56,858 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:56,858 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:C 2024-11-17T09:40:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:40:56,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. as already flushing 2024-11-17T09:40:56,922 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T09:40:56,922 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:40:56,922 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,922 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:40:56,932 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T09:40:56,932 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:40:56,932 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,932 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:56,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:40:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T09:40:57,203 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/ff17d434586c4596b8c254b887ee0e03 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ff17d434586c4596b8c254b887ee0e03 2024-11-17T09:40:57,210 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0fd42d9906a7b067f93ee39238302f5/A of c0fd42d9906a7b067f93ee39238302f5 into ff17d434586c4596b8c254b887ee0e03(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:40:57,210 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:57,210 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., storeName=c0fd42d9906a7b067f93ee39238302f5/A, priority=13, startTime=1731836456756; duration=0sec 2024-11-17T09:40:57,210 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:40:57,210 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0fd42d9906a7b067f93ee39238302f5:A 2024-11-17T09:40:57,243 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/3bc2414932b146ca8190d9b92bbcc747 2024-11-17T09:40:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/4ec9c65b8bc04607934c3ab44af7f691 is 50, key is test_row_0/B:col10/1731836456281/Put/seqid=0 2024-11-17T09:40:57,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741961_1137 (size=12301) 2024-11-17T09:40:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T09:40:57,658 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/4ec9c65b8bc04607934c3ab44af7f691 2024-11-17T09:40:57,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/f4c86fe3be1441e3908a466570829296 is 50, key is test_row_0/C:col10/1731836456281/Put/seqid=0 2024-11-17T09:40:57,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741962_1138 (size=12301) 2024-11-17T09:40:58,072 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/f4c86fe3be1441e3908a466570829296 2024-11-17T09:40:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/3bc2414932b146ca8190d9b92bbcc747 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3bc2414932b146ca8190d9b92bbcc747 2024-11-17T09:40:58,082 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3bc2414932b146ca8190d9b92bbcc747, entries=150, sequenceid=564, filesize=12.0 K 2024-11-17T09:40:58,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/4ec9c65b8bc04607934c3ab44af7f691 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4ec9c65b8bc04607934c3ab44af7f691 2024-11-17T09:40:58,087 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4ec9c65b8bc04607934c3ab44af7f691, entries=150, sequenceid=564, filesize=12.0 K 2024-11-17T09:40:58,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/f4c86fe3be1441e3908a466570829296 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f4c86fe3be1441e3908a466570829296 2024-11-17T09:40:58,092 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f4c86fe3be1441e3908a466570829296, entries=150, sequenceid=564, filesize=12.0 K 2024-11-17T09:40:58,093 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for c0fd42d9906a7b067f93ee39238302f5 in 1279ms, sequenceid=564, compaction requested=false 2024-11-17T09:40:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c0fd42d9906a7b067f93ee39238302f5: 2024-11-17T09:40:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:40:58,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T09:40:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T09:40:58,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T09:40:58,097 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5900 sec 2024-11-17T09:40:58,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.5970 sec 2024-11-17T09:40:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T09:40:58,650 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:40:59,891 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T09:40:59,891 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:40:59,891 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:59,892 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:40:59,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:00,092 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T09:41:00,092 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:00,092 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:00,092 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:00,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:00,101 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T09:41:00,101 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:00,101 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:00,101 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:00,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2605 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2604 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T09:41:00,101 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-17T09:41:00,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3534 rows 2024-11-17T09:41:00,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1178 2024-11-17T09:41:00,102 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3532 rows 2024-11-17T09:41:00,102 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:41:00,102 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:41:00,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:00,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:00,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:00,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T09:41:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T09:41:00,117 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836460117"}]},"ts":"1731836460117"} 2024-11-17T09:41:00,120 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T09:41:00,120 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T09:41:00,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:41:00,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, UNASSIGN}] 2024-11-17T09:41:00,128 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, UNASSIGN 2024-11-17T09:41:00,130 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c0fd42d9906a7b067f93ee39238302f5, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:00,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, UNASSIGN because future has completed 2024-11-17T09:41:00,134 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:41:00,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:41:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T09:41:00,291 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:41:00,292 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:41:00,292 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing c0fd42d9906a7b067f93ee39238302f5, disabling compactions & flushes 2024-11-17T09:41:00,292 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:41:00,292 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:41:00,292 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. after waiting 0 ms 2024-11-17T09:41:00,292 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:41:00,292 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing c0fd42d9906a7b067f93ee39238302f5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T09:41:00,293 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=A 2024-11-17T09:41:00,293 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:00,293 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=B 2024-11-17T09:41:00,293 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:00,293 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0fd42d9906a7b067f93ee39238302f5, store=C 2024-11-17T09:41:00,293 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:00,297 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/d54902d413124fd28ea7db22fdb691cc is 50, key is test_row_0/A:col10/1731836456930/Put/seqid=0 2024-11-17T09:41:00,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741963_1139 (size=12301) 2024-11-17T09:41:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T09:41:00,490 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T09:41:00,702 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/d54902d413124fd28ea7db22fdb691cc 2024-11-17T09:41:00,710 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/b9ff6a9a5d50457e87d068370fa112e3 is 50, key is test_row_0/B:col10/1731836456930/Put/seqid=0 2024-11-17T09:41:00,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741964_1140 (size=12301) 2024-11-17T09:41:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T09:41:01,115 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/b9ff6a9a5d50457e87d068370fa112e3 2024-11-17T09:41:01,138 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b64dc327b4fc412da7c85cf67e1816ba is 50, key is test_row_0/C:col10/1731836456930/Put/seqid=0 2024-11-17T09:41:01,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741965_1141 (size=12301) 2024-11-17T09:41:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T09:41:01,553 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b64dc327b4fc412da7c85cf67e1816ba 2024-11-17T09:41:01,558 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/A/d54902d413124fd28ea7db22fdb691cc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/d54902d413124fd28ea7db22fdb691cc 2024-11-17T09:41:01,563 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/d54902d413124fd28ea7db22fdb691cc, entries=150, sequenceid=575, filesize=12.0 K 2024-11-17T09:41:01,563 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/B/b9ff6a9a5d50457e87d068370fa112e3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b9ff6a9a5d50457e87d068370fa112e3 2024-11-17T09:41:01,567 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b9ff6a9a5d50457e87d068370fa112e3, entries=150, sequenceid=575, filesize=12.0 K 2024-11-17T09:41:01,568 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/.tmp/C/b64dc327b4fc412da7c85cf67e1816ba as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b64dc327b4fc412da7c85cf67e1816ba 2024-11-17T09:41:01,572 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b64dc327b4fc412da7c85cf67e1816ba, entries=150, sequenceid=575, filesize=12.0 K 2024-11-17T09:41:01,573 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0fd42d9906a7b067f93ee39238302f5 in 1281ms, sequenceid=575, compaction requested=true 2024-11-17T09:41:01,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/bd47a1529721499887bb51a3b6f0f618, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/6ccb974a4a944cae877de73a3df634fc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/407dbac55ff046489681615d54ebdd93, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0e32b856b21d4ab7b882285ed9b08b0a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/fb4b91d221fc4672a0c39653d34cdd25, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/eea55cb9f5e54cd48c92684630cca21c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/12305295f112466f8efea2150279f248, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3e8d4651f35e4e138b437a7ecdde1879, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/deae0acd305b49f4ba09b1a9e78a0a25, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0b345a979bfb4ee7bd8a34423ff5f830, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/112f509668a24dacb778a50f7dda54cd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/9c384d9cd2bc4b08a78d5a38458247de, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b30c4c281c1744e8836c1849aea7a8b1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e9fc2e969fed4c229a65d6a140f953bf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c0120d5a2777465b84cd2cd85bdcc69d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/8d7f0eb1aaac45f9809a7afbd449dd02, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/09c974307aca43eeb886ece777a8213e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b1f981979dae4ecf8614cda2bfa7f120, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c7c5f8c6f07b4065bf5de12dfc08e4af, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/1c05fc8c0023495f86c8dc922f0d17a7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ba44bae9f6064a0a9ab2fe7435ff0341, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/17033a6c7314472dbcbb604d49fbfaf8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a629078767f642c5848b47a736710cb6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/835a0c01bf05422194521ada73e87b2d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a9b01c26ae794ba2a933e078d2a7d5d9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/4418060699564e7493ac659677a918e1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/be7c6f96cc1d4fc290decae8ea07079f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c74f32d867924c70920d27a27f2f0096, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ecc83add2c974059be438eccaed7c201, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c8d94abc71db44b3a5483f5bc6de890d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/32d19c85198247dfaabaa159303aa5f4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/355aa5a7a68c42eea3985659cae82b2f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/45478d3881e740ac99c3b11b921a5423, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/895294d1add847f192a1d8cd2ab3852e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/7274ea1ed09f4999bd636c56ebc3e3b1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/aa04a3110eff4a5e9651ec0eb3c1e62e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/23a3dd27564344f1bec7a0fc7bc0c51d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c14467acb70c4e569f7687d7dc2ee6ab, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/f65dc85226c148d3b53e8e3b91dd67e4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e6e7a11ff97247d389c2e7efbd8236af] to archive 2024-11-17T09:41:01,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:41:01,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/bd47a1529721499887bb51a3b6f0f618 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/bd47a1529721499887bb51a3b6f0f618 2024-11-17T09:41:01,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/6ccb974a4a944cae877de73a3df634fc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/6ccb974a4a944cae877de73a3df634fc 2024-11-17T09:41:01,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/407dbac55ff046489681615d54ebdd93 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/407dbac55ff046489681615d54ebdd93 2024-11-17T09:41:01,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0e32b856b21d4ab7b882285ed9b08b0a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0e32b856b21d4ab7b882285ed9b08b0a 2024-11-17T09:41:01,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/fb4b91d221fc4672a0c39653d34cdd25 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/fb4b91d221fc4672a0c39653d34cdd25 2024-11-17T09:41:01,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/eea55cb9f5e54cd48c92684630cca21c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/eea55cb9f5e54cd48c92684630cca21c 2024-11-17T09:41:01,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/12305295f112466f8efea2150279f248 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/12305295f112466f8efea2150279f248 2024-11-17T09:41:01,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3e8d4651f35e4e138b437a7ecdde1879 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3e8d4651f35e4e138b437a7ecdde1879 2024-11-17T09:41:01,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/deae0acd305b49f4ba09b1a9e78a0a25 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/deae0acd305b49f4ba09b1a9e78a0a25 2024-11-17T09:41:01,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0b345a979bfb4ee7bd8a34423ff5f830 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/0b345a979bfb4ee7bd8a34423ff5f830 2024-11-17T09:41:01,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/112f509668a24dacb778a50f7dda54cd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/112f509668a24dacb778a50f7dda54cd 2024-11-17T09:41:01,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/9c384d9cd2bc4b08a78d5a38458247de to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/9c384d9cd2bc4b08a78d5a38458247de 2024-11-17T09:41:01,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b30c4c281c1744e8836c1849aea7a8b1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b30c4c281c1744e8836c1849aea7a8b1 2024-11-17T09:41:01,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e9fc2e969fed4c229a65d6a140f953bf to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e9fc2e969fed4c229a65d6a140f953bf 2024-11-17T09:41:01,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c0120d5a2777465b84cd2cd85bdcc69d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c0120d5a2777465b84cd2cd85bdcc69d 2024-11-17T09:41:01,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/8d7f0eb1aaac45f9809a7afbd449dd02 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/8d7f0eb1aaac45f9809a7afbd449dd02 2024-11-17T09:41:01,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/09c974307aca43eeb886ece777a8213e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/09c974307aca43eeb886ece777a8213e 2024-11-17T09:41:01,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b1f981979dae4ecf8614cda2bfa7f120 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/b1f981979dae4ecf8614cda2bfa7f120 2024-11-17T09:41:01,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c7c5f8c6f07b4065bf5de12dfc08e4af to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c7c5f8c6f07b4065bf5de12dfc08e4af 2024-11-17T09:41:01,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/1c05fc8c0023495f86c8dc922f0d17a7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/1c05fc8c0023495f86c8dc922f0d17a7 2024-11-17T09:41:01,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ba44bae9f6064a0a9ab2fe7435ff0341 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ba44bae9f6064a0a9ab2fe7435ff0341 2024-11-17T09:41:01,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/17033a6c7314472dbcbb604d49fbfaf8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/17033a6c7314472dbcbb604d49fbfaf8 2024-11-17T09:41:01,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a629078767f642c5848b47a736710cb6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a629078767f642c5848b47a736710cb6 2024-11-17T09:41:01,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/835a0c01bf05422194521ada73e87b2d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/835a0c01bf05422194521ada73e87b2d 2024-11-17T09:41:01,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a9b01c26ae794ba2a933e078d2a7d5d9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/a9b01c26ae794ba2a933e078d2a7d5d9 2024-11-17T09:41:01,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/4418060699564e7493ac659677a918e1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/4418060699564e7493ac659677a918e1 2024-11-17T09:41:01,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/be7c6f96cc1d4fc290decae8ea07079f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/be7c6f96cc1d4fc290decae8ea07079f 2024-11-17T09:41:01,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c74f32d867924c70920d27a27f2f0096 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c74f32d867924c70920d27a27f2f0096 2024-11-17T09:41:01,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ecc83add2c974059be438eccaed7c201 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ecc83add2c974059be438eccaed7c201 2024-11-17T09:41:01,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c8d94abc71db44b3a5483f5bc6de890d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c8d94abc71db44b3a5483f5bc6de890d 2024-11-17T09:41:01,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/32d19c85198247dfaabaa159303aa5f4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/32d19c85198247dfaabaa159303aa5f4 2024-11-17T09:41:01,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/355aa5a7a68c42eea3985659cae82b2f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/355aa5a7a68c42eea3985659cae82b2f 2024-11-17T09:41:01,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/45478d3881e740ac99c3b11b921a5423 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/45478d3881e740ac99c3b11b921a5423 2024-11-17T09:41:01,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/895294d1add847f192a1d8cd2ab3852e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/895294d1add847f192a1d8cd2ab3852e 2024-11-17T09:41:01,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/7274ea1ed09f4999bd636c56ebc3e3b1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/7274ea1ed09f4999bd636c56ebc3e3b1 2024-11-17T09:41:01,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/aa04a3110eff4a5e9651ec0eb3c1e62e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/aa04a3110eff4a5e9651ec0eb3c1e62e 2024-11-17T09:41:01,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/23a3dd27564344f1bec7a0fc7bc0c51d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/23a3dd27564344f1bec7a0fc7bc0c51d 2024-11-17T09:41:01,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c14467acb70c4e569f7687d7dc2ee6ab to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/c14467acb70c4e569f7687d7dc2ee6ab 2024-11-17T09:41:01,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/f65dc85226c148d3b53e8e3b91dd67e4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/f65dc85226c148d3b53e8e3b91dd67e4 2024-11-17T09:41:01,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e6e7a11ff97247d389c2e7efbd8236af to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/e6e7a11ff97247d389c2e7efbd8236af 2024-11-17T09:41:01,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/68a9cd6058c14bd6a7ae45749949c4db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3064624501454fa794812647c7a373c4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/06b9fc4ece5f4be58352bd799ee3d9eb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/75b26c5625ae4e899a2f81bead5ea937, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4abe95d49464ca5807674ccce103302, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b834a7cbfcb044b8ac5dc958560f7636, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d45ab3e1dcd444628e3ab0981d4dfa94, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/dac7252d90d244788276cc89b986b4e0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d84b7fbbc1a848f6b4e9003a90710690, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2faf1dc8d95243668a11c63a2850d728, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e0d083ba04d4457f972828532ecabd98, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/f65e4fb57a6842dbbf82d14304e4c8f9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/eae2cd0ccb6a49038a7e23c9b725307e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3088da1e4a244ffa9fb160f2755718f1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/96fa613796b449cdb6f8eb48f3656ae8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/7ae1058b72ca4370ad1eaf9b9fca1b35, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/21400e8361924df0b6a23cf558430f59, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fb2441b721ee4f1b9657bf6e8aa8f2f1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/a2bdd58f053b4336b271f817c144e4f8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e183bff307674dd3a465c5e1ae627bda, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4510bac6da254bbfa009e318c1d407c3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/6dad8d72dcf94e709d521d0b9020c828, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/9d0a4852b02f48aaa80ddc01229dfe71, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d70afd3edb124d17a4b28438d441f0fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/5e801b26a0a6462eaee966d3e0711042, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/1f49e40d778840ffb7487ee8888fa94e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4c1e80a6f434225b2d21e35b95fb9f4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/61600f48aa3d4d87b37b79797b800327, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/22a957bbe0f0472d96f0a93e3fbf668e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/64007493a9f3422a84892f60c59f0b45, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fccbe296536840228329b1da0396129d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/55faecd9aadc40a0a3a3964991b1b3d8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2a334c3b86134287aa1c52a1c6864401, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/07c6461814eb4953b9a339598c245c39, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/03b7a684c9614517a477672a91c31f90, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/ae4417d56b17460eb88320d62006a091, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8b6d71700652472da811e2b6e1f735a4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8ceef49027e64d2cbf07b9acb9345f1f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/710c71ff583b4d8aa54aa074a1cb73c8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/358c995859054af78cb5c8df622a4b1a] to archive 2024-11-17T09:41:01,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:41:01,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/68a9cd6058c14bd6a7ae45749949c4db to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/68a9cd6058c14bd6a7ae45749949c4db 2024-11-17T09:41:01,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3064624501454fa794812647c7a373c4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3064624501454fa794812647c7a373c4 2024-11-17T09:41:01,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/06b9fc4ece5f4be58352bd799ee3d9eb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/06b9fc4ece5f4be58352bd799ee3d9eb 2024-11-17T09:41:01,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/75b26c5625ae4e899a2f81bead5ea937 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/75b26c5625ae4e899a2f81bead5ea937 2024-11-17T09:41:01,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4abe95d49464ca5807674ccce103302 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4abe95d49464ca5807674ccce103302 2024-11-17T09:41:01,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b834a7cbfcb044b8ac5dc958560f7636 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b834a7cbfcb044b8ac5dc958560f7636 2024-11-17T09:41:01,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d45ab3e1dcd444628e3ab0981d4dfa94 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d45ab3e1dcd444628e3ab0981d4dfa94 2024-11-17T09:41:01,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/dac7252d90d244788276cc89b986b4e0 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/dac7252d90d244788276cc89b986b4e0 2024-11-17T09:41:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d84b7fbbc1a848f6b4e9003a90710690 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d84b7fbbc1a848f6b4e9003a90710690 2024-11-17T09:41:01,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2faf1dc8d95243668a11c63a2850d728 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2faf1dc8d95243668a11c63a2850d728 2024-11-17T09:41:01,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e0d083ba04d4457f972828532ecabd98 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e0d083ba04d4457f972828532ecabd98 2024-11-17T09:41:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/f65e4fb57a6842dbbf82d14304e4c8f9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/f65e4fb57a6842dbbf82d14304e4c8f9 2024-11-17T09:41:01,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/eae2cd0ccb6a49038a7e23c9b725307e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/eae2cd0ccb6a49038a7e23c9b725307e 2024-11-17T09:41:01,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3088da1e4a244ffa9fb160f2755718f1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/3088da1e4a244ffa9fb160f2755718f1 2024-11-17T09:41:01,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/96fa613796b449cdb6f8eb48f3656ae8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/96fa613796b449cdb6f8eb48f3656ae8 2024-11-17T09:41:01,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/7ae1058b72ca4370ad1eaf9b9fca1b35 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/7ae1058b72ca4370ad1eaf9b9fca1b35 2024-11-17T09:41:01,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/21400e8361924df0b6a23cf558430f59 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/21400e8361924df0b6a23cf558430f59 2024-11-17T09:41:01,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fb2441b721ee4f1b9657bf6e8aa8f2f1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fb2441b721ee4f1b9657bf6e8aa8f2f1 2024-11-17T09:41:01,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/a2bdd58f053b4336b271f817c144e4f8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/a2bdd58f053b4336b271f817c144e4f8 2024-11-17T09:41:01,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e183bff307674dd3a465c5e1ae627bda to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/e183bff307674dd3a465c5e1ae627bda 2024-11-17T09:41:01,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4510bac6da254bbfa009e318c1d407c3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4510bac6da254bbfa009e318c1d407c3 2024-11-17T09:41:01,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/6dad8d72dcf94e709d521d0b9020c828 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/6dad8d72dcf94e709d521d0b9020c828 2024-11-17T09:41:01,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/9d0a4852b02f48aaa80ddc01229dfe71 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/9d0a4852b02f48aaa80ddc01229dfe71 2024-11-17T09:41:01,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d70afd3edb124d17a4b28438d441f0fe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d70afd3edb124d17a4b28438d441f0fe 2024-11-17T09:41:01,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/5e801b26a0a6462eaee966d3e0711042 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/5e801b26a0a6462eaee966d3e0711042 2024-11-17T09:41:01,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/1f49e40d778840ffb7487ee8888fa94e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/1f49e40d778840ffb7487ee8888fa94e 2024-11-17T09:41:01,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4c1e80a6f434225b2d21e35b95fb9f4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/d4c1e80a6f434225b2d21e35b95fb9f4 2024-11-17T09:41:01,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/61600f48aa3d4d87b37b79797b800327 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/61600f48aa3d4d87b37b79797b800327 2024-11-17T09:41:01,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/22a957bbe0f0472d96f0a93e3fbf668e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/22a957bbe0f0472d96f0a93e3fbf668e 2024-11-17T09:41:01,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/64007493a9f3422a84892f60c59f0b45 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/64007493a9f3422a84892f60c59f0b45 2024-11-17T09:41:01,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fccbe296536840228329b1da0396129d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/fccbe296536840228329b1da0396129d 2024-11-17T09:41:01,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/55faecd9aadc40a0a3a3964991b1b3d8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/55faecd9aadc40a0a3a3964991b1b3d8 2024-11-17T09:41:01,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2a334c3b86134287aa1c52a1c6864401 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/2a334c3b86134287aa1c52a1c6864401 2024-11-17T09:41:01,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/07c6461814eb4953b9a339598c245c39 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/07c6461814eb4953b9a339598c245c39 2024-11-17T09:41:01,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/03b7a684c9614517a477672a91c31f90 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/03b7a684c9614517a477672a91c31f90 2024-11-17T09:41:01,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/ae4417d56b17460eb88320d62006a091 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/ae4417d56b17460eb88320d62006a091 2024-11-17T09:41:01,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8b6d71700652472da811e2b6e1f735a4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8b6d71700652472da811e2b6e1f735a4 2024-11-17T09:41:01,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8ceef49027e64d2cbf07b9acb9345f1f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/8ceef49027e64d2cbf07b9acb9345f1f 2024-11-17T09:41:01,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/710c71ff583b4d8aa54aa074a1cb73c8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/710c71ff583b4d8aa54aa074a1cb73c8 2024-11-17T09:41:01,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/358c995859054af78cb5c8df622a4b1a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/358c995859054af78cb5c8df622a4b1a 2024-11-17T09:41:01,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6334806e250449128af8558d43619f98, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6c937adae7ab421d9927d9a8a23e2117, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8d13a00fb7bb496baac3d6693bf57bc8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/84458f29c7914e2c87b3c500c41324f0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ab0fa5a7da5d48ac9d670e029b6cd66b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/0afa7bd656ed426789c008934af1234f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/c618d5fb96c047c3832df10d50db03b8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6ab16e43861c497bb6507d619e55bc21, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/42a93ee4f0974f428843ee4dd58ddafb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/decd7c7a3b0d4c5e8f299ba25e61ffa7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a04eaaa769974356baf32385c14db7ee, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f5edb8b975644392a661bfaccc61ded1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5c13b54f13dc4808a57b79f03acec084, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/d80277a10d494e66b0325fd916bf6628, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a4462690140043a79340062c9fb8697b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b0db3adf463f4460942d45333a0e26a4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6d0a5793c98d4a17ba7c39b742e5702e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/3f30ba861afb4ccda3f8dc9152c86848, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5349fa1c1590441fab54660ac7eca784, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ed6bf8b7ff0d4bd59b8b32dc4016f261, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a23e2fa7f72247c5b261bb0e36538a6a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/4c1536ed408449248a0376126686b22e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/7f8799e6e63e49cfa7d3bd066c64d5da, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5ac83d01f7934a1dab4aed51d3b1ac08, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dcf2f5f04d7b486b8e839ee8486d5e16, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5472af6a424a4d25aaf60bbf371425fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a01729ca152c4d3b9d153a2017900a1d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/69c739185c5e43289a3e218b150c9fa7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b1ba14b5cc134042a79d2fbd174ceca0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/df4f40664e0840b9a37f8880409dda22, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/fda25d00040b4208834581c05e33c174, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/78ace889cc4a4f71ba64adff1fc449e6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bc5133c6dc014701aaf80113b36ae50e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dd8b4809be8b49b587b054378cf472b1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/1a1402e1bb894dd4972eb26114fb002b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/202950d0af204454bf8c213e61842f13, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bdd22974f7fd4137b397c1d76ea46414, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6aed7c4be6ba4f11b3043aa4ecd0044d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/20edf59f0b784e3b82b04e8e70574500, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8050fddf1610405dbe5975cfd4b2d4fb] to archive 2024-11-17T09:41:01,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:41:01,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6334806e250449128af8558d43619f98 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6334806e250449128af8558d43619f98 2024-11-17T09:41:01,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6c937adae7ab421d9927d9a8a23e2117 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6c937adae7ab421d9927d9a8a23e2117 2024-11-17T09:41:01,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8d13a00fb7bb496baac3d6693bf57bc8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8d13a00fb7bb496baac3d6693bf57bc8 2024-11-17T09:41:01,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/84458f29c7914e2c87b3c500c41324f0 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/84458f29c7914e2c87b3c500c41324f0 2024-11-17T09:41:01,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ab0fa5a7da5d48ac9d670e029b6cd66b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ab0fa5a7da5d48ac9d670e029b6cd66b 2024-11-17T09:41:01,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/0afa7bd656ed426789c008934af1234f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/0afa7bd656ed426789c008934af1234f 2024-11-17T09:41:01,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/c618d5fb96c047c3832df10d50db03b8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/c618d5fb96c047c3832df10d50db03b8 2024-11-17T09:41:01,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6ab16e43861c497bb6507d619e55bc21 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6ab16e43861c497bb6507d619e55bc21 2024-11-17T09:41:01,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/42a93ee4f0974f428843ee4dd58ddafb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/42a93ee4f0974f428843ee4dd58ddafb 2024-11-17T09:41:01,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/decd7c7a3b0d4c5e8f299ba25e61ffa7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/decd7c7a3b0d4c5e8f299ba25e61ffa7 2024-11-17T09:41:01,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a04eaaa769974356baf32385c14db7ee to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a04eaaa769974356baf32385c14db7ee 2024-11-17T09:41:01,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f5edb8b975644392a661bfaccc61ded1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f5edb8b975644392a661bfaccc61ded1 2024-11-17T09:41:01,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5c13b54f13dc4808a57b79f03acec084 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5c13b54f13dc4808a57b79f03acec084 2024-11-17T09:41:01,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/d80277a10d494e66b0325fd916bf6628 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/d80277a10d494e66b0325fd916bf6628 2024-11-17T09:41:01,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a4462690140043a79340062c9fb8697b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a4462690140043a79340062c9fb8697b 2024-11-17T09:41:01,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b0db3adf463f4460942d45333a0e26a4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b0db3adf463f4460942d45333a0e26a4 2024-11-17T09:41:01,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6d0a5793c98d4a17ba7c39b742e5702e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6d0a5793c98d4a17ba7c39b742e5702e 2024-11-17T09:41:01,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/3f30ba861afb4ccda3f8dc9152c86848 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/3f30ba861afb4ccda3f8dc9152c86848 2024-11-17T09:41:01,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5349fa1c1590441fab54660ac7eca784 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5349fa1c1590441fab54660ac7eca784 2024-11-17T09:41:01,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ed6bf8b7ff0d4bd59b8b32dc4016f261 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/ed6bf8b7ff0d4bd59b8b32dc4016f261 2024-11-17T09:41:01,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a23e2fa7f72247c5b261bb0e36538a6a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a23e2fa7f72247c5b261bb0e36538a6a 2024-11-17T09:41:01,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/4c1536ed408449248a0376126686b22e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/4c1536ed408449248a0376126686b22e 2024-11-17T09:41:01,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/7f8799e6e63e49cfa7d3bd066c64d5da to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/7f8799e6e63e49cfa7d3bd066c64d5da 2024-11-17T09:41:01,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5ac83d01f7934a1dab4aed51d3b1ac08 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5ac83d01f7934a1dab4aed51d3b1ac08 2024-11-17T09:41:01,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dcf2f5f04d7b486b8e839ee8486d5e16 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dcf2f5f04d7b486b8e839ee8486d5e16 2024-11-17T09:41:01,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5472af6a424a4d25aaf60bbf371425fe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/5472af6a424a4d25aaf60bbf371425fe 2024-11-17T09:41:01,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a01729ca152c4d3b9d153a2017900a1d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/a01729ca152c4d3b9d153a2017900a1d 2024-11-17T09:41:01,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/69c739185c5e43289a3e218b150c9fa7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/69c739185c5e43289a3e218b150c9fa7 2024-11-17T09:41:01,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b1ba14b5cc134042a79d2fbd174ceca0 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b1ba14b5cc134042a79d2fbd174ceca0 2024-11-17T09:41:01,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/df4f40664e0840b9a37f8880409dda22 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/df4f40664e0840b9a37f8880409dda22 2024-11-17T09:41:01,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/fda25d00040b4208834581c05e33c174 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/fda25d00040b4208834581c05e33c174 2024-11-17T09:41:01,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/78ace889cc4a4f71ba64adff1fc449e6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/78ace889cc4a4f71ba64adff1fc449e6 2024-11-17T09:41:01,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bc5133c6dc014701aaf80113b36ae50e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bc5133c6dc014701aaf80113b36ae50e 2024-11-17T09:41:01,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dd8b4809be8b49b587b054378cf472b1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/dd8b4809be8b49b587b054378cf472b1 2024-11-17T09:41:01,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/1a1402e1bb894dd4972eb26114fb002b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/1a1402e1bb894dd4972eb26114fb002b 2024-11-17T09:41:01,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/202950d0af204454bf8c213e61842f13 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/202950d0af204454bf8c213e61842f13 2024-11-17T09:41:01,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bdd22974f7fd4137b397c1d76ea46414 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/bdd22974f7fd4137b397c1d76ea46414 2024-11-17T09:41:01,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6aed7c4be6ba4f11b3043aa4ecd0044d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/6aed7c4be6ba4f11b3043aa4ecd0044d 2024-11-17T09:41:01,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/20edf59f0b784e3b82b04e8e70574500 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/20edf59f0b784e3b82b04e8e70574500 2024-11-17T09:41:01,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8050fddf1610405dbe5975cfd4b2d4fb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/8050fddf1610405dbe5975cfd4b2d4fb 2024-11-17T09:41:01,745 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/recovered.edits/578.seqid, newMaxSeqId=578, maxSeqId=1 2024-11-17T09:41:01,745 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. 2024-11-17T09:41:01,746 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for c0fd42d9906a7b067f93ee39238302f5: Waiting for close lock at 1731836460292Running coprocessor pre-close hooks at 1731836460292Disabling compacts and flushes for region at 1731836460292Disabling writes for close at 1731836460292Obtaining lock to block concurrent updates at 1731836460292Preparing flush snapshotting stores in c0fd42d9906a7b067f93ee39238302f5 at 1731836460292Finished memstore snapshotting TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731836460293 (+1 ms)Flushing stores of TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5. at 1731836460293Flushing c0fd42d9906a7b067f93ee39238302f5/A: creating writer at 1731836460294 (+1 ms)Flushing c0fd42d9906a7b067f93ee39238302f5/A: appending metadata at 1731836460296 (+2 ms)Flushing c0fd42d9906a7b067f93ee39238302f5/A: closing flushed file at 1731836460297 (+1 ms)Flushing c0fd42d9906a7b067f93ee39238302f5/B: creating writer at 1731836460706 (+409 ms)Flushing c0fd42d9906a7b067f93ee39238302f5/B: appending metadata at 1731836460709 (+3 ms)Flushing c0fd42d9906a7b067f93ee39238302f5/B: closing flushed file at 1731836460709Flushing c0fd42d9906a7b067f93ee39238302f5/C: creating writer at 1731836461126 (+417 ms)Flushing c0fd42d9906a7b067f93ee39238302f5/C: appending metadata at 1731836461137 (+11 ms)Flushing c0fd42d9906a7b067f93ee39238302f5/C: closing flushed file at 1731836461137Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c56bcbf: reopening flushed file at 1731836461558 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b1abf74: reopening flushed file at 1731836461563 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@713c4a27: reopening flushed file at 1731836461567 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0fd42d9906a7b067f93ee39238302f5 in 1281ms, sequenceid=575, compaction requested=true at 1731836461573 (+6 ms)Writing region close event to WAL at 1731836461741 (+168 ms)Running coprocessor post-close hooks at 1731836461745 (+4 ms)Closed at 1731836461745 2024-11-17T09:41:01,748 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:41:01,749 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c0fd42d9906a7b067f93ee39238302f5, regionState=CLOSED 2024-11-17T09:41:01,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:41:01,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T09:41:01,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure c0fd42d9906a7b067f93ee39238302f5, server=70e37c475432,40671,1731836432285 in 1.6180 sec 2024-11-17T09:41:01,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-17T09:41:01,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0fd42d9906a7b067f93ee39238302f5, UNASSIGN in 1.6270 sec 2024-11-17T09:41:01,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T09:41:01,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6340 sec 2024-11-17T09:41:01,759 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836461759"}]},"ts":"1731836461759"} 2024-11-17T09:41:01,760 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T09:41:01,760 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T09:41:01,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6520 sec 2024-11-17T09:41:01,952 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T09:41:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T09:41:02,260 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:02,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T09:41:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:02,269 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T09:41:02,271 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:02,278 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:41:02,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/recovered.edits] 2024-11-17T09:41:02,288 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3bc2414932b146ca8190d9b92bbcc747 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/3bc2414932b146ca8190d9b92bbcc747 2024-11-17T09:41:02,290 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/d54902d413124fd28ea7db22fdb691cc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/d54902d413124fd28ea7db22fdb691cc 2024-11-17T09:41:02,294 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ff17d434586c4596b8c254b887ee0e03 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/A/ff17d434586c4596b8c254b887ee0e03 2024-11-17T09:41:02,298 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/305d3e3f51e24bd4a822399ae97cdb19 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/305d3e3f51e24bd4a822399ae97cdb19 2024-11-17T09:41:02,301 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4ec9c65b8bc04607934c3ab44af7f691 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/4ec9c65b8bc04607934c3ab44af7f691 2024-11-17T09:41:02,303 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b9ff6a9a5d50457e87d068370fa112e3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/B/b9ff6a9a5d50457e87d068370fa112e3 2024-11-17T09:41:02,306 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/40f4e54a8a2b46b697ed47d3f8067789 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/40f4e54a8a2b46b697ed47d3f8067789 2024-11-17T09:41:02,308 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b64dc327b4fc412da7c85cf67e1816ba to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/b64dc327b4fc412da7c85cf67e1816ba 2024-11-17T09:41:02,310 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f4c86fe3be1441e3908a466570829296 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/C/f4c86fe3be1441e3908a466570829296 2024-11-17T09:41:02,314 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/recovered.edits/578.seqid to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5/recovered.edits/578.seqid 2024-11-17T09:41:02,315 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/c0fd42d9906a7b067f93ee39238302f5 2024-11-17T09:41:02,315 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T09:41:02,323 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:02,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T09:41:02,346 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T09:41:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T09:41:02,397 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T09:41:02,401 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:02,402 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T09:41:02,402 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731836462402"}]},"ts":"9223372036854775807"} 2024-11-17T09:41:02,413 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T09:41:02,413 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c0fd42d9906a7b067f93ee39238302f5, NAME => 'TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T09:41:02,414 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T09:41:02,416 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731836462414"}]},"ts":"9223372036854775807"} 2024-11-17T09:41:02,422 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T09:41:02,423 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:02,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 158 msec 2024-11-17T09:41:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T09:41:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T09:41:02,591 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:02,633 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=246 (was 209) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/70e37c475432:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;70e37c475432:40671-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1569465998_22 at /127.0.0.1:36098 [Waiting for operation #282] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/70e37c475432:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1838368081_22 at /127.0.0.1:35920 [Waiting for operation #304] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=441 (was 268) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3126 (was 3711) 2024-11-17T09:41:02,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=441, ProcessCount=11, AvailableMemoryMB=3124 2024-11-17T09:41:02,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:41:02,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:41:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:02,656 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:41:02,657 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:02,658 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:41:02,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-17T09:41:02,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T09:41:02,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741966_1142 (size=1061) 2024-11-17T09:41:02,714 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:41:02,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741967_1143 (size=53) 2024-11-17T09:41:02,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:41:02,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 78abf17b9dde42332f107c0d3469140f, disabling compactions & flushes 2024-11-17T09:41:02,732 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:02,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:02,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. after waiting 0 ms 2024-11-17T09:41:02,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:02,732 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:02,732 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 78abf17b9dde42332f107c0d3469140f: Waiting for close lock at 1731836462732Disabling compacts and flushes for region at 1731836462732Disabling writes for close at 1731836462732Writing region close event to WAL at 1731836462732Closed at 1731836462732 2024-11-17T09:41:02,734 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:41:02,735 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731836462734"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836462734"}]},"ts":"1731836462734"} 2024-11-17T09:41:02,737 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:41:02,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:41:02,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836462739"}]},"ts":"1731836462739"} 2024-11-17T09:41:02,741 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T09:41:02,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, ASSIGN}] 2024-11-17T09:41:02,744 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, ASSIGN 2024-11-17T09:41:02,744 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, ASSIGN; state=OFFLINE, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=false 2024-11-17T09:41:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T09:41:02,895 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:02,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, ASSIGN because future has completed 2024-11-17T09:41:02,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:41:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T09:41:03,054 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,054 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:41:03,055 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,055 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:41:03,055 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,055 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,056 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,057 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:03,058 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78abf17b9dde42332f107c0d3469140f columnFamilyName A 2024-11-17T09:41:03,058 DEBUG [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:03,059 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(327): Store=78abf17b9dde42332f107c0d3469140f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:03,059 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,060 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:03,060 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78abf17b9dde42332f107c0d3469140f columnFamilyName B 2024-11-17T09:41:03,060 DEBUG [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:03,061 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(327): Store=78abf17b9dde42332f107c0d3469140f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:03,061 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,062 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:03,062 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78abf17b9dde42332f107c0d3469140f columnFamilyName C 2024-11-17T09:41:03,062 DEBUG [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:03,062 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(327): Store=78abf17b9dde42332f107c0d3469140f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:03,062 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,062 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,063 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,063 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,064 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,065 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,065 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:41:03,066 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,069 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:41:03,069 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 78abf17b9dde42332f107c0d3469140f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59350513, jitterRate=-0.11560843884944916}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:41:03,069 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,070 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 78abf17b9dde42332f107c0d3469140f: Running coprocessor pre-open hook at 1731836463055Writing region info on filesystem at 1731836463055Initializing all the Stores at 1731836463056 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836463056Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836463056Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836463056Cleaning up temporary data from old regions at 1731836463065 (+9 ms)Running coprocessor post-open hooks at 1731836463069 (+4 ms)Region opened successfully at 1731836463070 (+1 ms) 2024-11-17T09:41:03,071 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., pid=32, masterSystemTime=1731836463051 2024-11-17T09:41:03,073 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,073 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:03,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:41:03,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T09:41:03,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 in 178 msec 2024-11-17T09:41:03,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-17T09:41:03,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, ASSIGN in 336 msec 2024-11-17T09:41:03,081 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:41:03,081 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836463081"}]},"ts":"1731836463081"} 2024-11-17T09:41:03,082 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T09:41:03,083 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:41:03,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 430 msec 2024-11-17T09:41:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T09:41:03,290 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:03,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328db2e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:03,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:03,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:03,293 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:03,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f855e65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:03,295 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:03,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655d0d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:03,296 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:03,298 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:03,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:41:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,44533,1731836431485 2024-11-17T09:41:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ed75f2b 2024-11-17T09:41:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:41:03,301 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:41:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:41:03,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:41:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:03,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741968_1144 (size=1097) 2024-11-17T09:41:03,726 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T09:41:03,727 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T09:41:03,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:41:03,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, REOPEN/MOVE}] 2024-11-17T09:41:03,742 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, REOPEN/MOVE 2024-11-17T09:41:03,743 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:03,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, REOPEN/MOVE because future has completed 2024-11-17T09:41:03,746 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:41:03,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:41:03,899 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,899 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:41:03,900 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 78abf17b9dde42332f107c0d3469140f, disabling compactions & flushes 2024-11-17T09:41:03,900 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,900 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,900 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. after waiting 0 ms 2024-11-17T09:41:03,900 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,905 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T09:41:03,906 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:03,906 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 78abf17b9dde42332f107c0d3469140f: Waiting for close lock at 1731836463899Running coprocessor pre-close hooks at 1731836463899Disabling compacts and flushes for region at 1731836463899Disabling writes for close at 1731836463900 (+1 ms)Writing region close event to WAL at 1731836463900Running coprocessor post-close hooks at 1731836463906 (+6 ms)Closed at 1731836463906 2024-11-17T09:41:03,906 WARN [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 78abf17b9dde42332f107c0d3469140f to self. 2024-11-17T09:41:03,909 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=CLOSED 2024-11-17T09:41:03,910 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:03,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:41:03,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T09:41:03,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 in 167 msec 2024-11-17T09:41:03,916 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, REOPEN/MOVE; state=CLOSED, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=true 2024-11-17T09:41:04,067 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, REOPEN/MOVE because future has completed 2024-11-17T09:41:04,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:41:04,225 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,225 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:41:04,225 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,225 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:41:04,226 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,226 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,230 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,231 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:04,237 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78abf17b9dde42332f107c0d3469140f columnFamilyName A 2024-11-17T09:41:04,239 DEBUG [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:04,240 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(327): Store=78abf17b9dde42332f107c0d3469140f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:04,241 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,241 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:04,242 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78abf17b9dde42332f107c0d3469140f columnFamilyName B 2024-11-17T09:41:04,242 DEBUG [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:04,242 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(327): Store=78abf17b9dde42332f107c0d3469140f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:04,242 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,243 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:04,243 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78abf17b9dde42332f107c0d3469140f columnFamilyName C 2024-11-17T09:41:04,243 DEBUG [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:04,244 INFO [StoreOpener-78abf17b9dde42332f107c0d3469140f-1 {}] regionserver.HStore(327): Store=78abf17b9dde42332f107c0d3469140f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:04,244 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,244 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,245 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,246 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,247 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,247 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,248 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:41:04,250 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,251 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 78abf17b9dde42332f107c0d3469140f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75093020, jitterRate=0.11897319555282593}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:41:04,251 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,252 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 78abf17b9dde42332f107c0d3469140f: Running coprocessor pre-open hook at 1731836464226Writing region info on filesystem at 1731836464226Initializing all the Stores at 1731836464227 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836464227Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836464227Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836464227Cleaning up temporary data from old regions at 1731836464247 (+20 ms)Running coprocessor post-open hooks at 1731836464251 (+4 ms)Region opened successfully at 1731836464252 (+1 ms) 2024-11-17T09:41:04,253 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., pid=37, masterSystemTime=1731836464221 2024-11-17T09:41:04,255 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,255 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,255 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=OPEN, openSeqNum=5, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:41:04,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-17T09:41:04,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 in 188 msec 2024-11-17T09:41:04,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-17T09:41:04,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, REOPEN/MOVE in 519 msec 2024-11-17T09:41:04,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-17T09:41:04,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-11-17T09:41:04,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-11-17T09:41:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T09:41:04,271 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:04,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263f8ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d09e208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,275 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2e1884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,278 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,280 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e60e4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,282 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27645d2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,284 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4993e8fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,286 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c841903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d73364e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b469573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713fe79b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,294 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1006e0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,295 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@815cb27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536e4975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ac8dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,300 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6f899f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,301 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5fc009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:04,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:04,303 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:04,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679b330, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:04,305 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b24a261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:04,306 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,306 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,306 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,307 DEBUG [Thread-722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,307 DEBUG [Thread-724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,309 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,312 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-17T09:41:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T09:41:04,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:04,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,332 DEBUG [Thread-718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,333 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,328 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:04,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:04,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:04,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:41:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:41:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:04,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:04,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f213aadc65b24e8d942f5233dae9f4f1_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836464338/Put/seqid=0 2024-11-17T09:41:04,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836524412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836524413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731836464382, max timestamp=1731836464419Num uniques 859; 2024-11-17T09:41:04,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-17T09:41:04,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731836464382, max timestamp=1731836464419Num uniques 859; 2024-11-17T09:41:04,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-17T09:41:04,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T09:41:04,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731836464382, max timestamp=1731836464419Num uniques 859; 2024-11-17T09:41:04,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-17T09:41:04,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741969_1145 (size=12154) 2024-11-17T09:41:04,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:04,463 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f213aadc65b24e8d942f5233dae9f4f1_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f213aadc65b24e8d942f5233dae9f4f1_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:04,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/dd44eb9e67e74118a58a36ebc0bd46fc, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:04,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/dd44eb9e67e74118a58a36ebc0bd46fc is 175, key is test_row_0/A:col10/1731836464338/Put/seqid=0 2024-11-17T09:41:04,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741970_1146 (size=30955) 2024-11-17T09:41:04,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/dd44eb9e67e74118a58a36ebc0bd46fc 2024-11-17T09:41:04,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836524527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836524530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836524530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836524530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836524530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/01475e5f4cdb402cb528be44c4dd5e89 is 50, key is test_row_0/B:col10/1731836464338/Put/seqid=0 2024-11-17T09:41:04,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741971_1147 (size=12001) 2024-11-17T09:41:04,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T09:41:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836524650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836524651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836524661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836524760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836524761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836524870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836524871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836524871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:04,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:04,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:04,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T09:41:05,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/01475e5f4cdb402cb528be44c4dd5e89 2024-11-17T09:41:05,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7663613b5d8e47eea18d79ef4ee10045 is 50, key is test_row_0/C:col10/1731836464338/Put/seqid=0 2024-11-17T09:41:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741972_1148 (size=12001) 2024-11-17T09:41:05,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7663613b5d8e47eea18d79ef4ee10045 2024-11-17T09:41:05,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/dd44eb9e67e74118a58a36ebc0bd46fc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/dd44eb9e67e74118a58a36ebc0bd46fc 2024-11-17T09:41:05,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/dd44eb9e67e74118a58a36ebc0bd46fc, entries=150, sequenceid=14, filesize=30.2 K 2024-11-17T09:41:05,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/01475e5f4cdb402cb528be44c4dd5e89 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/01475e5f4cdb402cb528be44c4dd5e89 2024-11-17T09:41:05,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836525070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836525070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/01475e5f4cdb402cb528be44c4dd5e89, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T09:41:05,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7663613b5d8e47eea18d79ef4ee10045 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7663613b5d8e47eea18d79ef4ee10045 2024-11-17T09:41:05,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7663613b5d8e47eea18d79ef4ee10045, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T09:41:05,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 78abf17b9dde42332f107c0d3469140f in 759ms, sequenceid=14, compaction requested=false 2024-11-17T09:41:05,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:05,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=161.02 KB heapSize=436.55 KB 2024-11-17T09:41:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:05,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836525196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836525206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d368ee5276244a9b91eadf1914a24026_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836465191/Put/seqid=0 2024-11-17T09:41:05,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836525204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741973_1149 (size=17034) 2024-11-17T09:41:05,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836525310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836525319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836525321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:05,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:05,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T09:41:05,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836525519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836525530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836525530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:05,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:05,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836525590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836525590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,647 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d368ee5276244a9b91eadf1914a24026_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d368ee5276244a9b91eadf1914a24026_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:05,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0d0ce32f8be14a92adcfe30b56e6f553, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:05,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0d0ce32f8be14a92adcfe30b56e6f553 is 175, key is test_row_0/A:col10/1731836465191/Put/seqid=0 2024-11-17T09:41:05,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741974_1150 (size=48139) 2024-11-17T09:41:05,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0d0ce32f8be14a92adcfe30b56e6f553 2024-11-17T09:41:05,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/830b261e970a4779ae27657049b6e4a3 is 50, key is test_row_0/B:col10/1731836465191/Put/seqid=0 2024-11-17T09:41:05,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:05,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:05,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:05,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741975_1151 (size=12001) 2024-11-17T09:41:05,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/830b261e970a4779ae27657049b6e4a3 2024-11-17T09:41:05,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/2966bdca11e948bebed671230279ecc7 is 50, key is test_row_0/C:col10/1731836465191/Put/seqid=0 2024-11-17T09:41:05,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741976_1152 (size=12001) 2024-11-17T09:41:05,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/2966bdca11e948bebed671230279ecc7 2024-11-17T09:41:05,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0d0ce32f8be14a92adcfe30b56e6f553 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0d0ce32f8be14a92adcfe30b56e6f553 2024-11-17T09:41:05,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836525830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:05,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0d0ce32f8be14a92adcfe30b56e6f553, entries=250, sequenceid=51, filesize=47.0 K 2024-11-17T09:41:05,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/830b261e970a4779ae27657049b6e4a3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/830b261e970a4779ae27657049b6e4a3 2024-11-17T09:41:05,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/830b261e970a4779ae27657049b6e4a3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T09:41:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/2966bdca11e948bebed671230279ecc7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/2966bdca11e948bebed671230279ecc7 2024-11-17T09:41:05,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/2966bdca11e948bebed671230279ecc7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T09:41:05,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~476.34 KB/487776, currentSize=13.42 KB/13740 for 78abf17b9dde42332f107c0d3469140f in 673ms, sequenceid=51, compaction requested=false 2024-11-17T09:41:05,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T09:41:05,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:05,878 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-17T09:41:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117065e2e079c46475ba0813edcad631029_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_1/A:col10/1731836465200/Put/seqid=0 2024-11-17T09:41:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741977_1153 (size=9714) 2024-11-17T09:41:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1731836466181, max timestamp=1731836466274Num uniques 859; 2024-11-17T09:41:06,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1731836466181, max timestamp=1731836466274Num uniques 859; 2024-11-17T09:41:06,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-17T09:41:06,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-17T09:41:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1731836466181, max timestamp=1731836466274Num uniques 859; 2024-11-17T09:41:06,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-17T09:41:06,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836526323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836526325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836526340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:06,372 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117065e2e079c46475ba0813edcad631029_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117065e2e079c46475ba0813edcad631029_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:06,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/d025896a0cf64fdbbb9759f2dc3d1a32, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/d025896a0cf64fdbbb9759f2dc3d1a32 is 175, key is test_row_1/A:col10/1731836465200/Put/seqid=0 2024-11-17T09:41:06,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741978_1154 (size=22361) 2024-11-17T09:41:06,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836526440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836526440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T09:41:06,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836526600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,607 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836526610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,614 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836526650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836526650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,796 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/d025896a0cf64fdbbb9759f2dc3d1a32 2024-11-17T09:41:06,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e05fd011f97f4ac4b4c0fd84af51e20a is 50, key is test_row_1/B:col10/1731836465200/Put/seqid=0 2024-11-17T09:41:06,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741979_1155 (size=9657) 2024-11-17T09:41:06,870 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e05fd011f97f4ac4b4c0fd84af51e20a 2024-11-17T09:41:06,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/0c9f8ad027f64fd7a892642a6de7ddf1 is 50, key is test_row_1/C:col10/1731836465200/Put/seqid=0 2024-11-17T09:41:06,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741980_1156 (size=9657) 2024-11-17T09:41:06,907 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/0c9f8ad027f64fd7a892642a6de7ddf1 2024-11-17T09:41:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/d025896a0cf64fdbbb9759f2dc3d1a32 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/d025896a0cf64fdbbb9759f2dc3d1a32 2024-11-17T09:41:06,922 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/d025896a0cf64fdbbb9759f2dc3d1a32, entries=100, sequenceid=56, filesize=21.8 K 2024-11-17T09:41:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e05fd011f97f4ac4b4c0fd84af51e20a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e05fd011f97f4ac4b4c0fd84af51e20a 2024-11-17T09:41:06,938 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e05fd011f97f4ac4b4c0fd84af51e20a, entries=100, sequenceid=56, filesize=9.4 K 2024-11-17T09:41:06,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/0c9f8ad027f64fd7a892642a6de7ddf1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/0c9f8ad027f64fd7a892642a6de7ddf1 2024-11-17T09:41:06,947 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/0c9f8ad027f64fd7a892642a6de7ddf1, entries=100, sequenceid=56, filesize=9.4 K 2024-11-17T09:41:06,949 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=181.14 KB/185490 for 78abf17b9dde42332f107c0d3469140f in 1071ms, sequenceid=56, compaction requested=true 2024-11-17T09:41:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:06,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-17T09:41:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-17T09:41:06,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-17T09:41:06,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-17T09:41:06,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.6420 sec 2024-11-17T09:41:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:06,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=187.85 KB heapSize=497.48 KB 2024-11-17T09:41:06,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:06,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:06,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:06,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:06,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:06,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836526973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836526976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ea00dab177d1458b83fe5366f9f9bc26_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836466320/Put/seqid=0 2024-11-17T09:41:07,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741981_1157 (size=14594) 2024-11-17T09:41:07,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:07,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ea00dab177d1458b83fe5366f9f9bc26_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ea00dab177d1458b83fe5366f9f9bc26_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:07,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b2a29a88906a4c88ad91c1d2ce2c5a63, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:07,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b2a29a88906a4c88ad91c1d2ce2c5a63 is 175, key is test_row_0/A:col10/1731836466320/Put/seqid=0 2024-11-17T09:41:07,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741982_1158 (size=39549) 2024-11-17T09:41:07,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836527090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:07,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836527090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b2a29a88906a4c88ad91c1d2ce2c5a63 2024-11-17T09:41:07,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/97cfa73cc30a4015b7a0ce9bf6b819e0 is 50, key is test_row_0/B:col10/1731836466320/Put/seqid=0 2024-11-17T09:41:07,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741983_1159 (size=12001) 2024-11-17T09:41:07,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836527300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:07,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836527300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,348 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:41:07,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836527360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:07,361 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/97cfa73cc30a4015b7a0ce9bf6b819e0 2024-11-17T09:41:07,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/d27b146564284b6399650ab746c054ae is 50, key is test_row_0/C:col10/1731836466320/Put/seqid=0 2024-11-17T09:41:07,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836527610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836527610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:07,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741984_1160 (size=12001) 2024-11-17T09:41:08,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/d27b146564284b6399650ab746c054ae 2024-11-17T09:41:08,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b2a29a88906a4c88ad91c1d2ce2c5a63 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b2a29a88906a4c88ad91c1d2ce2c5a63 2024-11-17T09:41:08,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b2a29a88906a4c88ad91c1d2ce2c5a63, entries=200, sequenceid=91, filesize=38.6 K 2024-11-17T09:41:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/97cfa73cc30a4015b7a0ce9bf6b819e0 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/97cfa73cc30a4015b7a0ce9bf6b819e0 2024-11-17T09:41:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/97cfa73cc30a4015b7a0ce9bf6b819e0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T09:41:08,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/d27b146564284b6399650ab746c054ae as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d27b146564284b6399650ab746c054ae 2024-11-17T09:41:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d27b146564284b6399650ab746c054ae, entries=150, sequenceid=91, filesize=11.7 K 2024-11-17T09:41:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~497.44 KB/509376, currentSize=6.71 KB/6870 for 78abf17b9dde42332f107c0d3469140f in 1106ms, sequenceid=91, compaction requested=true 2024-11-17T09:41:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:08,079 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:08,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:08,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,079 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:08,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:08,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:08,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:08,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,082 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:08,082 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:08,082 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,083 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/dd44eb9e67e74118a58a36ebc0bd46fc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0d0ce32f8be14a92adcfe30b56e6f553, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/d025896a0cf64fdbbb9759f2dc3d1a32, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b2a29a88906a4c88ad91c1d2ce2c5a63] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=137.7 K 2024-11-17T09:41:08,083 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,083 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/dd44eb9e67e74118a58a36ebc0bd46fc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0d0ce32f8be14a92adcfe30b56e6f553, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/d025896a0cf64fdbbb9759f2dc3d1a32, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b2a29a88906a4c88ad91c1d2ce2c5a63] 2024-11-17T09:41:08,083 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:08,084 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:08,084 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,084 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/01475e5f4cdb402cb528be44c4dd5e89, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/830b261e970a4779ae27657049b6e4a3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e05fd011f97f4ac4b4c0fd84af51e20a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/97cfa73cc30a4015b7a0ce9bf6b819e0] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=44.6 K 2024-11-17T09:41:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,084 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd44eb9e67e74118a58a36ebc0bd46fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731836464337 2024-11-17T09:41:08,084 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 01475e5f4cdb402cb528be44c4dd5e89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731836464337 2024-11-17T09:41:08,085 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0ce32f8be14a92adcfe30b56e6f553, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731836464472 2024-11-17T09:41:08,085 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 830b261e970a4779ae27657049b6e4a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731836465188 2024-11-17T09:41:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,085 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d025896a0cf64fdbbb9759f2dc3d1a32, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731836465200 2024-11-17T09:41:08,085 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e05fd011f97f4ac4b4c0fd84af51e20a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731836465200 2024-11-17T09:41:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,086 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2a29a88906a4c88ad91c1d2ce2c5a63, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731836466313 2024-11-17T09:41:08,086 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 97cfa73cc30a4015b7a0ce9bf6b819e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731836466320 2024-11-17T09:41:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,109 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,119 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,120 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/4c44670137d14fe4bfed210c13399776 is 50, key is test_row_0/B:col10/1731836466320/Put/seqid=0 2024-11-17T09:41:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,132 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a34b8b3b9dc4439686554b42868090b5_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,135 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a34b8b3b9dc4439686554b42868090b5_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,135 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a34b8b3b9dc4439686554b42868090b5_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:41:08,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:08,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:08,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:08,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:08,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:08,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:08,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741985_1161 (size=12139) 2024-11-17T09:41:08,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741986_1162 (size=4469) 2024-11-17T09:41:08,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d05d9144908b42dc96bfbaa94d536227_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836468155/Put/seqid=0 2024-11-17T09:41:08,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836468215, max timestamp=1731836468239Num uniques 859; 2024-11-17T09:41:08,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:08,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741987_1163 (size=12154) 2024-11-17T09:41:08,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836468215, max timestamp=1731836468239Num uniques 859; 2024-11-17T09:41:08,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:08,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836468215, max timestamp=1731836468239Num uniques 859; 2024-11-17T09:41:08,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:08,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836528296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836528298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836528410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836528410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T09:41:08,500 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:08,500 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-17T09:41:08,503 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T09:41:08,503 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:08,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:08,578 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/4c44670137d14fe4bfed210c13399776 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4c44670137d14fe4bfed210c13399776 2024-11-17T09:41:08,586 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into 4c44670137d14fe4bfed210c13399776(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:08,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:08,586 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=12, startTime=1731836468079; duration=0sec 2024-11-17T09:41:08,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:08,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:08,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:08,588 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:08,588 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:08,588 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,588 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7663613b5d8e47eea18d79ef4ee10045, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/2966bdca11e948bebed671230279ecc7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/0c9f8ad027f64fd7a892642a6de7ddf1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d27b146564284b6399650ab746c054ae] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=44.6 K 2024-11-17T09:41:08,589 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7663613b5d8e47eea18d79ef4ee10045, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731836464337 2024-11-17T09:41:08,590 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2966bdca11e948bebed671230279ecc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731836465188 2024-11-17T09:41:08,590 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c9f8ad027f64fd7a892642a6de7ddf1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731836465200 2024-11-17T09:41:08,591 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d27b146564284b6399650ab746c054ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731836466320 2024-11-17T09:41:08,603 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:08,604 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/425c2c6acb034905a868271758c3ff44 is 50, key is test_row_0/C:col10/1731836466320/Put/seqid=0 2024-11-17T09:41:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T09:41:08,613 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#141 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:08,614 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0cef4a08c46340efb6feb2aec26b35a1 is 175, key is test_row_0/A:col10/1731836466320/Put/seqid=0 2024-11-17T09:41:08,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741988_1164 (size=12139) 2024-11-17T09:41:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836528620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836528620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836528630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,635 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836528631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,636 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,640 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/425c2c6acb034905a868271758c3ff44 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/425c2c6acb034905a868271758c3ff44 2024-11-17T09:41:08,647 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:08,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741989_1165 (size=31093) 2024-11-17T09:41:08,650 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into 425c2c6acb034905a868271758c3ff44(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:08,651 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:08,651 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=12, startTime=1731836468080; duration=0sec 2024-11-17T09:41:08,651 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:08,651 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:08,653 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d05d9144908b42dc96bfbaa94d536227_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d05d9144908b42dc96bfbaa94d536227_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:08,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T09:41:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,664 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0cef4a08c46340efb6feb2aec26b35a1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0cef4a08c46340efb6feb2aec26b35a1 2024-11-17T09:41:08,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/faddbbbd37df47dfb9cbbc9cda781fbd, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:08,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/faddbbbd37df47dfb9cbbc9cda781fbd is 175, key is test_row_0/A:col10/1731836468155/Put/seqid=0 2024-11-17T09:41:08,673 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into 0cef4a08c46340efb6feb2aec26b35a1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:08,673 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:08,673 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=12, startTime=1731836468078; duration=0sec 2024-11-17T09:41:08,673 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:08,673 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:08,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741990_1166 (size=30955) 2024-11-17T09:41:08,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/faddbbbd37df47dfb9cbbc9cda781fbd 2024-11-17T09:41:08,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/ce272bed9f1a423db049e8e14ca2b0b1 is 50, key is test_row_0/B:col10/1731836468155/Put/seqid=0 2024-11-17T09:41:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741991_1167 (size=12001) 2024-11-17T09:41:08,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/ce272bed9f1a423db049e8e14ca2b0b1 2024-11-17T09:41:08,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c736bfabc80147d4b04a299fc0a65869 is 50, key is test_row_0/C:col10/1731836468155/Put/seqid=0 2024-11-17T09:41:08,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741992_1168 (size=12001) 2024-11-17T09:41:08,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T09:41:08,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:08,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T09:41:08,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836528930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836528931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:08,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:08,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T09:41:08,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:08,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:08,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:09,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c736bfabc80147d4b04a299fc0a65869 2024-11-17T09:41:09,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T09:41:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:09,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:09,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:09,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:09,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/faddbbbd37df47dfb9cbbc9cda781fbd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/faddbbbd37df47dfb9cbbc9cda781fbd 2024-11-17T09:41:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T09:41:09,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/faddbbbd37df47dfb9cbbc9cda781fbd, entries=150, sequenceid=101, filesize=30.2 K 2024-11-17T09:41:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/ce272bed9f1a423db049e8e14ca2b0b1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/ce272bed9f1a423db049e8e14ca2b0b1 2024-11-17T09:41:09,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/ce272bed9f1a423db049e8e14ca2b0b1, entries=150, sequenceid=101, filesize=11.7 K 2024-11-17T09:41:09,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c736bfabc80147d4b04a299fc0a65869 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c736bfabc80147d4b04a299fc0a65869 2024-11-17T09:41:09,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c736bfabc80147d4b04a299fc0a65869, entries=150, sequenceid=101, filesize=11.7 K 2024-11-17T09:41:09,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 78abf17b9dde42332f107c0d3469140f in 981ms, sequenceid=101, compaction requested=false 2024-11-17T09:41:09,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:09,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T09:41:09,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:09,281 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-17T09:41:09,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:09,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:09,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:09,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ebee650d799b4622b304eedad1cd28c3_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836468296/Put/seqid=0 2024-11-17T09:41:09,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741993_1169 (size=12304) 2024-11-17T09:41:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:09,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:09,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836529420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:09,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836529440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836529440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836529530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T09:41:09,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:09,713 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ebee650d799b4622b304eedad1cd28c3_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ebee650d799b4622b304eedad1cd28c3_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/ed70c255906b464899c2384183b49632, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:09,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/ed70c255906b464899c2384183b49632 is 175, key is test_row_0/A:col10/1731836468296/Put/seqid=0 2024-11-17T09:41:09,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741994_1170 (size=31105) 2024-11-17T09:41:09,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836529740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836530050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,119 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/ed70c255906b464899c2384183b49632 2024-11-17T09:41:10,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/a5ab610edeb845a1870816ab224f0f3a is 50, key is test_row_0/B:col10/1731836468296/Put/seqid=0 2024-11-17T09:41:10,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741995_1171 (size=12151) 2024-11-17T09:41:10,143 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/a5ab610edeb845a1870816ab224f0f3a 2024-11-17T09:41:10,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/3d128dcec5654828947b64a9ecbbe491 is 50, key is test_row_0/C:col10/1731836468296/Put/seqid=0 2024-11-17T09:41:10,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741996_1172 (size=12151) 2024-11-17T09:41:10,160 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/3d128dcec5654828947b64a9ecbbe491 2024-11-17T09:41:10,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/ed70c255906b464899c2384183b49632 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/ed70c255906b464899c2384183b49632 2024-11-17T09:41:10,175 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/ed70c255906b464899c2384183b49632, entries=150, sequenceid=141, filesize=30.4 K 2024-11-17T09:41:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/a5ab610edeb845a1870816ab224f0f3a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a5ab610edeb845a1870816ab224f0f3a 2024-11-17T09:41:10,182 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a5ab610edeb845a1870816ab224f0f3a, entries=150, sequenceid=141, filesize=11.9 K 2024-11-17T09:41:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/3d128dcec5654828947b64a9ecbbe491 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/3d128dcec5654828947b64a9ecbbe491 2024-11-17T09:41:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,190 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/3d128dcec5654828947b64a9ecbbe491, entries=150, sequenceid=141, filesize=11.9 K 2024-11-17T09:41:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,191 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 78abf17b9dde42332f107c0d3469140f in 910ms, sequenceid=141, compaction requested=true 2024-11-17T09:41:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:10,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-17T09:41:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-17T09:41:10,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-17T09:41:10,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6900 sec 2024-11-17T09:41:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6970 sec 2024-11-17T09:41:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T09:41:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:10,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e61dda1b0fb4b4383b319b81b3e1b3b_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836470463/Put/seqid=0 2024-11-17T09:41:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741997_1173 (size=12304) 2024-11-17T09:41:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:10,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172e61dda1b0fb4b4383b319b81b3e1b3b_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e61dda1b0fb4b4383b319b81b3e1b3b_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:10,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c99d6ab9241f420abdbe6b5bbf02ca78, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:10,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c99d6ab9241f420abdbe6b5bbf02ca78 is 175, key is test_row_0/A:col10/1731836470463/Put/seqid=0 2024-11-17T09:41:10,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741998_1174 (size=31101) 2024-11-17T09:41:10,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836470521, max timestamp=1731836470556Num uniques 859; 2024-11-17T09:41:10,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:10,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836470521, max timestamp=1731836470556Num uniques 859; 2024-11-17T09:41:10,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836470521, max timestamp=1731836470556Num uniques 859; 2024-11-17T09:41:10,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:10,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:10,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c99d6ab9241f420abdbe6b5bbf02ca78 2024-11-17T09:41:10,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/a63329cae8c84fd481a8aac2512196f7 is 50, key is test_row_0/B:col10/1731836470463/Put/seqid=0 2024-11-17T09:41:10,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741999_1175 (size=9757) 2024-11-17T09:41:10,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836530619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836530619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836530624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T09:41:10,651 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:10,651 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T09:41:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T09:41:10,655 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:10,659 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:10,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:10,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836530730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836530730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836530739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T09:41:10,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T09:41:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:10,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:10,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836530939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836530940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836530950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:10,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T09:41:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:10,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T09:41:10,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/a63329cae8c84fd481a8aac2512196f7 2024-11-17T09:41:11,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/1ef639077bdf40878cfc5d5a0b62d92c is 50, key is test_row_0/C:col10/1731836470463/Put/seqid=0 2024-11-17T09:41:11,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742000_1176 (size=9757) 2024-11-17T09:41:11,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/1ef639077bdf40878cfc5d5a0b62d92c 2024-11-17T09:41:11,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c99d6ab9241f420abdbe6b5bbf02ca78 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c99d6ab9241f420abdbe6b5bbf02ca78 2024-11-17T09:41:11,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c99d6ab9241f420abdbe6b5bbf02ca78, entries=150, sequenceid=152, filesize=30.4 K 2024-11-17T09:41:11,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/a63329cae8c84fd481a8aac2512196f7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a63329cae8c84fd481a8aac2512196f7 2024-11-17T09:41:11,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a63329cae8c84fd481a8aac2512196f7, entries=100, sequenceid=152, filesize=9.5 K 2024-11-17T09:41:11,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/1ef639077bdf40878cfc5d5a0b62d92c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1ef639077bdf40878cfc5d5a0b62d92c 2024-11-17T09:41:11,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1ef639077bdf40878cfc5d5a0b62d92c, entries=100, sequenceid=152, filesize=9.5 K 2024-11-17T09:41:11,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 78abf17b9dde42332f107c0d3469140f in 572ms, sequenceid=152, compaction requested=true 2024-11-17T09:41:11,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:11,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:11,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:11,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:11,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:11,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:11,038 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:11,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:41:11,039 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:11,040 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124254 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:11,040 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:11,040 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,041 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0cef4a08c46340efb6feb2aec26b35a1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/faddbbbd37df47dfb9cbbc9cda781fbd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/ed70c255906b464899c2384183b49632, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c99d6ab9241f420abdbe6b5bbf02ca78] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=121.3 K 2024-11-17T09:41:11,041 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,041 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0cef4a08c46340efb6feb2aec26b35a1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/faddbbbd37df47dfb9cbbc9cda781fbd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/ed70c255906b464899c2384183b49632, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c99d6ab9241f420abdbe6b5bbf02ca78] 2024-11-17T09:41:11,041 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:11,041 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:11,041 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,041 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cef4a08c46340efb6feb2aec26b35a1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731836466320 2024-11-17T09:41:11,041 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4c44670137d14fe4bfed210c13399776, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/ce272bed9f1a423db049e8e14ca2b0b1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a5ab610edeb845a1870816ab224f0f3a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a63329cae8c84fd481a8aac2512196f7] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=45.0 K 2024-11-17T09:41:11,042 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting faddbbbd37df47dfb9cbbc9cda781fbd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731836468155 2024-11-17T09:41:11,042 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c44670137d14fe4bfed210c13399776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731836466320 2024-11-17T09:41:11,042 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ce272bed9f1a423db049e8e14ca2b0b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731836468155 2024-11-17T09:41:11,042 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed70c255906b464899c2384183b49632, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836468288 2024-11-17T09:41:11,043 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ab610edeb845a1870816ab224f0f3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836468288 2024-11-17T09:41:11,043 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99d6ab9241f420abdbe6b5bbf02ca78, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731836469408 2024-11-17T09:41:11,043 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a63329cae8c84fd481a8aac2512196f7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731836469408 2024-11-17T09:41:11,083 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:11,084 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/d62fbe6e57e14b1e9fe40e2e762c9bdf is 50, key is test_row_0/B:col10/1731836470463/Put/seqid=0 2024-11-17T09:41:11,088 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:11,101 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117652436e749d046cfa4192c5e33dded94_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:11,104 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117652436e749d046cfa4192c5e33dded94_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:11,104 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117652436e749d046cfa4192c5e33dded94_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:11,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742001_1177 (size=12425) 2024-11-17T09:41:11,117 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/d62fbe6e57e14b1e9fe40e2e762c9bdf as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d62fbe6e57e14b1e9fe40e2e762c9bdf 2024-11-17T09:41:11,122 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into d62fbe6e57e14b1e9fe40e2e762c9bdf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:11,122 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:11,123 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=12, startTime=1731836471038; duration=0sec 2024-11-17T09:41:11,123 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:11,123 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:11,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T09:41:11,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,125 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-17T09:41:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:11,126 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:11,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742002_1178 (size=4469) 2024-11-17T09:41:11,129 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#154 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:11,130 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c45b594b3ddb48018205be4836267226 is 175, key is test_row_0/A:col10/1731836470463/Put/seqid=0 2024-11-17T09:41:11,134 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:11,134 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:11,134 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,134 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/425c2c6acb034905a868271758c3ff44, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c736bfabc80147d4b04a299fc0a65869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/3d128dcec5654828947b64a9ecbbe491, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1ef639077bdf40878cfc5d5a0b62d92c] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=45.0 K 2024-11-17T09:41:11,135 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 425c2c6acb034905a868271758c3ff44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731836466320 2024-11-17T09:41:11,136 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c736bfabc80147d4b04a299fc0a65869, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731836468155 2024-11-17T09:41:11,137 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d128dcec5654828947b64a9ecbbe491, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836468288 2024-11-17T09:41:11,137 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ef639077bdf40878cfc5d5a0b62d92c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731836469408 2024-11-17T09:41:11,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f2b8b6dc8e2049de97497e9056b839b7_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836470616/Put/seqid=0 2024-11-17T09:41:11,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742003_1179 (size=31486) 2024-11-17T09:41:11,167 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:11,168 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/30998ba8db1041e7ada8992ad1fcb234 is 50, key is test_row_0/C:col10/1731836470463/Put/seqid=0 2024-11-17T09:41:11,176 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c45b594b3ddb48018205be4836267226 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c45b594b3ddb48018205be4836267226 2024-11-17T09:41:11,186 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into c45b594b3ddb48018205be4836267226(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:11,186 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:11,186 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=12, startTime=1731836471038; duration=0sec 2024-11-17T09:41:11,186 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:11,186 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:11,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742005_1181 (size=12425) 2024-11-17T09:41:11,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742004_1180 (size=12304) 2024-11-17T09:41:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:11,207 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f2b8b6dc8e2049de97497e9056b839b7_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f2b8b6dc8e2049de97497e9056b839b7_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:11,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b04909572dbe4f649fbe801ce1f13dca, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:11,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b04909572dbe4f649fbe801ce1f13dca is 175, key is test_row_0/A:col10/1731836470616/Put/seqid=0 2024-11-17T09:41:11,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742006_1182 (size=31105) 2024-11-17T09:41:11,231 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b04909572dbe4f649fbe801ce1f13dca 2024-11-17T09:41:11,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/874e58042b224c73a1aef6ad4a3a9610 is 50, key is test_row_0/B:col10/1731836470616/Put/seqid=0 2024-11-17T09:41:11,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:11,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:11,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742007_1183 (size=12151) 2024-11-17T09:41:11,263 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/874e58042b224c73a1aef6ad4a3a9610 2024-11-17T09:41:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/6383867b0828405cad5da18af759b480 is 50, key is test_row_0/C:col10/1731836470616/Put/seqid=0 2024-11-17T09:41:11,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836531272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742008_1184 (size=12151) 2024-11-17T09:41:11,281 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/6383867b0828405cad5da18af759b480 2024-11-17T09:41:11,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836531278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/b04909572dbe4f649fbe801ce1f13dca as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b04909572dbe4f649fbe801ce1f13dca 2024-11-17T09:41:11,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836531280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T09:41:11,290 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b04909572dbe4f649fbe801ce1f13dca, entries=150, sequenceid=187, filesize=30.4 K 2024-11-17T09:41:11,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/874e58042b224c73a1aef6ad4a3a9610 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/874e58042b224c73a1aef6ad4a3a9610 2024-11-17T09:41:11,296 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/874e58042b224c73a1aef6ad4a3a9610, entries=150, sequenceid=187, filesize=11.9 K 2024-11-17T09:41:11,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/6383867b0828405cad5da18af759b480 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/6383867b0828405cad5da18af759b480 2024-11-17T09:41:11,301 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/6383867b0828405cad5da18af759b480, entries=150, sequenceid=187, filesize=11.9 K 2024-11-17T09:41:11,302 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 78abf17b9dde42332f107c0d3469140f in 177ms, sequenceid=187, compaction requested=false 2024-11-17T09:41:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T09:41:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T09:41:11,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T09:41:11,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 645 msec 2024-11-17T09:41:11,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 657 msec 2024-11-17T09:41:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:11,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T09:41:11,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:11,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:11,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:11,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:11,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:11,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:11,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e3c834b4236f4f999aa4dabfadabcc5b_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836471269/Put/seqid=0 2024-11-17T09:41:11,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742009_1185 (size=12304) 2024-11-17T09:41:11,428 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:11,435 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e3c834b4236f4f999aa4dabfadabcc5b_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e3c834b4236f4f999aa4dabfadabcc5b_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:11,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/f6b321e0c56f4d6189db4750ca57d99f, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:11,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/f6b321e0c56f4d6189db4750ca57d99f is 175, key is test_row_0/A:col10/1731836471269/Put/seqid=0 2024-11-17T09:41:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836531433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836531444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836531447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742010_1186 (size=31105) 2024-11-17T09:41:11,480 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/f6b321e0c56f4d6189db4750ca57d99f 2024-11-17T09:41:11,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/1754a5ff0174443cbe1d0d639fadc282 is 50, key is test_row_0/B:col10/1731836471269/Put/seqid=0 2024-11-17T09:41:11,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742011_1187 (size=12151) 2024-11-17T09:41:11,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836531550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836531560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836531561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,599 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/30998ba8db1041e7ada8992ad1fcb234 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/30998ba8db1041e7ada8992ad1fcb234 2024-11-17T09:41:11,608 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into 30998ba8db1041e7ada8992ad1fcb234(size=12.1 K), total size for store is 24 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:11,608 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:11,608 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=12, startTime=1731836471038; duration=0sec 2024-11-17T09:41:11,609 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:11,609 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836531770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836531770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836531770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:11,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:11,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T09:41:11,800 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:11,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T09:41:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T09:41:11,802 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:11,803 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:11,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:11,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/1754a5ff0174443cbe1d0d639fadc282 2024-11-17T09:41:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T09:41:11,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/24202ad268844db69d788d45b88b70f9 is 50, key is test_row_0/C:col10/1731836471269/Put/seqid=0 2024-11-17T09:41:11,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742012_1188 (size=12151) 2024-11-17T09:41:11,952 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T09:41:11,952 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T09:41:11,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T09:41:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:11,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:11,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:11,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:12,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836532080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836532080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836532080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T09:41:12,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:12,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:12,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T09:41:12,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T09:41:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:12,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:12,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/24202ad268844db69d788d45b88b70f9 2024-11-17T09:41:12,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/f6b321e0c56f4d6189db4750ca57d99f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/f6b321e0c56f4d6189db4750ca57d99f 2024-11-17T09:41:12,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/f6b321e0c56f4d6189db4750ca57d99f, entries=150, sequenceid=202, filesize=30.4 K 2024-11-17T09:41:12,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/1754a5ff0174443cbe1d0d639fadc282 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/1754a5ff0174443cbe1d0d639fadc282 2024-11-17T09:41:12,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/1754a5ff0174443cbe1d0d639fadc282, entries=150, sequenceid=202, filesize=11.9 K 2024-11-17T09:41:12,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/24202ad268844db69d788d45b88b70f9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/24202ad268844db69d788d45b88b70f9 2024-11-17T09:41:12,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/24202ad268844db69d788d45b88b70f9, entries=150, sequenceid=202, filesize=11.9 K 2024-11-17T09:41:12,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 78abf17b9dde42332f107c0d3469140f in 958ms, sequenceid=202, compaction requested=true 2024-11-17T09:41:12,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:12,352 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:12,353 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93696 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:12,353 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:12,353 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:12,353 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c45b594b3ddb48018205be4836267226, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b04909572dbe4f649fbe801ce1f13dca, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/f6b321e0c56f4d6189db4750ca57d99f] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=91.5 K 2024-11-17T09:41:12,353 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:12,353 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,353 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c45b594b3ddb48018205be4836267226, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b04909572dbe4f649fbe801ce1f13dca, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/f6b321e0c56f4d6189db4750ca57d99f] 2024-11-17T09:41:12,354 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45b594b3ddb48018205be4836267226, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731836468288 2024-11-17T09:41:12,354 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04909572dbe4f649fbe801ce1f13dca, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731836470597 2024-11-17T09:41:12,355 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b321e0c56f4d6189db4750ca57d99f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731836471253 2024-11-17T09:41:12,355 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:12,355 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:12,355 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,355 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d62fbe6e57e14b1e9fe40e2e762c9bdf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/874e58042b224c73a1aef6ad4a3a9610, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/1754a5ff0174443cbe1d0d639fadc282] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=35.9 K 2024-11-17T09:41:12,356 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d62fbe6e57e14b1e9fe40e2e762c9bdf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731836468288 2024-11-17T09:41:12,356 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 874e58042b224c73a1aef6ad4a3a9610, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731836470597 2024-11-17T09:41:12,357 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1754a5ff0174443cbe1d0d639fadc282, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731836471253 2024-11-17T09:41:12,370 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:12,376 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117eeac510ca33a40c1ba748fdccfcd66eb_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:12,377 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:12,377 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117eeac510ca33a40c1ba748fdccfcd66eb_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:12,378 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eeac510ca33a40c1ba748fdccfcd66eb_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:12,378 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e479f5f6ae844cdb8cfd1de590b4cd3d is 50, key is test_row_0/B:col10/1731836471269/Put/seqid=0 2024-11-17T09:41:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742013_1189 (size=12527) 2024-11-17T09:41:12,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742014_1190 (size=4469) 2024-11-17T09:41:12,400 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#162 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:12,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/6aece789623041dd855eff99054770e4 is 175, key is test_row_0/A:col10/1731836471269/Put/seqid=0 2024-11-17T09:41:12,405 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e479f5f6ae844cdb8cfd1de590b4cd3d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e479f5f6ae844cdb8cfd1de590b4cd3d 2024-11-17T09:41:12,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742015_1191 (size=31481) 2024-11-17T09:41:12,416 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into e479f5f6ae844cdb8cfd1de590b4cd3d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:12,416 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:12,416 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=13, startTime=1731836472353; duration=0sec 2024-11-17T09:41:12,416 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:12,416 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:12,416 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:12,418 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:12,418 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:12,418 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,418 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/30998ba8db1041e7ada8992ad1fcb234, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/6383867b0828405cad5da18af759b480, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/24202ad268844db69d788d45b88b70f9] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=35.9 K 2024-11-17T09:41:12,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T09:41:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,419 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T09:41:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:12,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:12,421 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 30998ba8db1041e7ada8992ad1fcb234, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731836468288 2024-11-17T09:41:12,421 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6383867b0828405cad5da18af759b480, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731836470597 2024-11-17T09:41:12,422 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 24202ad268844db69d788d45b88b70f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731836471253 2024-11-17T09:41:12,425 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/6aece789623041dd855eff99054770e4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/6aece789623041dd855eff99054770e4 2024-11-17T09:41:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T09:41:12,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177535c91b2c3f4b009a53b2789d274494_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836471416/Put/seqid=0 2024-11-17T09:41:12,441 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into 6aece789623041dd855eff99054770e4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:12,441 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:12,441 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=13, startTime=1731836472351; duration=0sec 2024-11-17T09:41:12,441 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:12,441 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:12,459 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:12,460 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/fe4c1f701431461d8832b4f29ccf9007 is 50, key is test_row_0/C:col10/1731836471269/Put/seqid=0 2024-11-17T09:41:12,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742016_1192 (size=12304) 2024-11-17T09:41:12,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:12,468 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177535c91b2c3f4b009a53b2789d274494_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177535c91b2c3f4b009a53b2789d274494_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/e603478be9aa4bf5a9d54d42f220eb96, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:12,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/e603478be9aa4bf5a9d54d42f220eb96 is 175, key is test_row_0/A:col10/1731836471416/Put/seqid=0 2024-11-17T09:41:12,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742017_1193 (size=12527) 2024-11-17T09:41:12,517 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/fe4c1f701431461d8832b4f29ccf9007 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/fe4c1f701431461d8832b4f29ccf9007 2024-11-17T09:41:12,524 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into fe4c1f701431461d8832b4f29ccf9007(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:12,524 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:12,524 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=13, startTime=1731836472353; duration=0sec 2024-11-17T09:41:12,524 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:12,524 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:12,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742018_1194 (size=31105) 2024-11-17T09:41:12,535 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/e603478be9aa4bf5a9d54d42f220eb96 2024-11-17T09:41:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/942d8d7ead34424eb94b30b556742c5e is 50, key is test_row_0/B:col10/1731836471416/Put/seqid=0 2024-11-17T09:41:12,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742019_1195 (size=12151) 2024-11-17T09:41:12,580 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/942d8d7ead34424eb94b30b556742c5e 2024-11-17T09:41:12,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/121ebb11f86f4dd5a03001e8a957f924 is 50, key is test_row_0/C:col10/1731836471416/Put/seqid=0 2024-11-17T09:41:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:12,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:12,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742020_1196 (size=12151) 2024-11-17T09:41:12,615 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/121ebb11f86f4dd5a03001e8a957f924 2024-11-17T09:41:12,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/e603478be9aa4bf5a9d54d42f220eb96 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/e603478be9aa4bf5a9d54d42f220eb96 2024-11-17T09:41:12,628 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/e603478be9aa4bf5a9d54d42f220eb96, entries=150, sequenceid=222, filesize=30.4 K 2024-11-17T09:41:12,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/942d8d7ead34424eb94b30b556742c5e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/942d8d7ead34424eb94b30b556742c5e 2024-11-17T09:41:12,635 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/942d8d7ead34424eb94b30b556742c5e, entries=150, sequenceid=222, filesize=11.9 K 2024-11-17T09:41:12,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/121ebb11f86f4dd5a03001e8a957f924 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/121ebb11f86f4dd5a03001e8a957f924 2024-11-17T09:41:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836532628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,641 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/121ebb11f86f4dd5a03001e8a957f924, entries=150, sequenceid=222, filesize=11.9 K 2024-11-17T09:41:12,642 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 78abf17b9dde42332f107c0d3469140f in 223ms, sequenceid=222, compaction requested=false 2024-11-17T09:41:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:12,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T09:41:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:12,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T09:41:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:12,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T09:41:12,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T09:41:12,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 844 msec 2024-11-17T09:41:12,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 850 msec 2024-11-17T09:41:12,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111776eae4e583f84d54a0b7b28f77a57ef7_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836472610/Put/seqid=0 2024-11-17T09:41:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836532671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836532675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836532679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836532678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742021_1197 (size=14794) 2024-11-17T09:41:12,723 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:12,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111776eae4e583f84d54a0b7b28f77a57ef7_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111776eae4e583f84d54a0b7b28f77a57ef7_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:12,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/3613cb0db4f74133bac4105354ae50c3, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:12,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/3613cb0db4f74133bac4105354ae50c3 is 175, key is test_row_0/A:col10/1731836472610/Put/seqid=0 2024-11-17T09:41:12,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742022_1198 (size=39749) 2024-11-17T09:41:12,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/3613cb0db4f74133bac4105354ae50c3 2024-11-17T09:41:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836532749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/d8cc0bd88d3d4bc4895df7cd33dc72df is 50, key is test_row_0/B:col10/1731836472610/Put/seqid=0 2024-11-17T09:41:12,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836532790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836532790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836532790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836532790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:12,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742023_1199 (size=12151) 2024-11-17T09:41:12,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/d8cc0bd88d3d4bc4895df7cd33dc72df 2024-11-17T09:41:12,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/08caa9be016247b880ece16c5f2d0a45 is 50, key is test_row_0/C:col10/1731836472610/Put/seqid=0 2024-11-17T09:41:12,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742024_1200 (size=12151) 2024-11-17T09:41:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T09:41:12,940 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:12,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T09:41:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T09:41:12,944 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:12,945 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:12,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836532960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836533000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836533000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836533000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836533000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T09:41:13,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:13,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:13,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T09:41:13,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/08caa9be016247b880ece16c5f2d0a45 2024-11-17T09:41:13,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836533269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/3613cb0db4f74133bac4105354ae50c3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/3613cb0db4f74133bac4105354ae50c3 2024-11-17T09:41:13,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/3613cb0db4f74133bac4105354ae50c3, entries=200, sequenceid=239, filesize=38.8 K 2024-11-17T09:41:13,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/d8cc0bd88d3d4bc4895df7cd33dc72df as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d8cc0bd88d3d4bc4895df7cd33dc72df 2024-11-17T09:41:13,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d8cc0bd88d3d4bc4895df7cd33dc72df, entries=150, sequenceid=239, filesize=11.9 K 2024-11-17T09:41:13,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/08caa9be016247b880ece16c5f2d0a45 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/08caa9be016247b880ece16c5f2d0a45 2024-11-17T09:41:13,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/08caa9be016247b880ece16c5f2d0a45, entries=150, sequenceid=239, filesize=11.9 K 2024-11-17T09:41:13,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 78abf17b9dde42332f107c0d3469140f in 652ms, sequenceid=239, compaction requested=true 2024-11-17T09:41:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:13,296 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:13,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:41:13,296 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:13,301 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:13,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:13,301 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:13,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:13,301 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,301 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e479f5f6ae844cdb8cfd1de590b4cd3d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/942d8d7ead34424eb94b30b556742c5e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d8cc0bd88d3d4bc4895df7cd33dc72df] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=36.0 K 2024-11-17T09:41:13,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/6aece789623041dd855eff99054770e4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/e603478be9aa4bf5a9d54d42f220eb96, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/3613cb0db4f74133bac4105354ae50c3] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=99.9 K 2024-11-17T09:41:13,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,302 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/6aece789623041dd855eff99054770e4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/e603478be9aa4bf5a9d54d42f220eb96, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/3613cb0db4f74133bac4105354ae50c3] 2024-11-17T09:41:13,302 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e479f5f6ae844cdb8cfd1de590b4cd3d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731836471253 2024-11-17T09:41:13,302 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aece789623041dd855eff99054770e4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731836471253 2024-11-17T09:41:13,303 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 942d8d7ead34424eb94b30b556742c5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731836471416 2024-11-17T09:41:13,303 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e603478be9aa4bf5a9d54d42f220eb96, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731836471416 2024-11-17T09:41:13,303 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d8cc0bd88d3d4bc4895df7cd33dc72df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731836472610 2024-11-17T09:41:13,303 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3613cb0db4f74133bac4105354ae50c3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731836472610 2024-11-17T09:41:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:13,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T09:41:13,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:13,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:13,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:13,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:13,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:13,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:13,317 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:13,323 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#172 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:13,324 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/c8ebc4424f7749dfb92eefb89df21b1c is 50, key is test_row_0/B:col10/1731836472610/Put/seqid=0 2024-11-17T09:41:13,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836533337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836533339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,350 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117dc461b44fb204120a3368691e66e26d2_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:13,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836533340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836533341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,352 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117dc461b44fb204120a3368691e66e26d2_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:13,352 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dc461b44fb204120a3368691e66e26d2_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:13,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177e70411a108c42f0bfefe0732d6798fc_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836472677/Put/seqid=0 2024-11-17T09:41:13,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742025_1201 (size=12629) 2024-11-17T09:41:13,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:13,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742026_1202 (size=4469) 2024-11-17T09:41:13,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742027_1203 (size=12404) 2024-11-17T09:41:13,433 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:13,439 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177e70411a108c42f0bfefe0732d6798fc_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177e70411a108c42f0bfefe0732d6798fc_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:13,440 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19119bb619e449ed8cb88be2383fad0c, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:13,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19119bb619e449ed8cb88be2383fad0c is 175, key is test_row_0/A:col10/1731836472677/Put/seqid=0 2024-11-17T09:41:13,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836533450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742028_1204 (size=31205) 2024-11-17T09:41:13,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836533460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836533460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836533461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:13,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:13,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T09:41:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836533660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836533680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836533680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836533680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836533780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:13,826 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#171 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:13,826 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/9e0f4da92ac94b15b04a418f8740d9f7 is 175, key is test_row_0/A:col10/1731836472610/Put/seqid=0 2024-11-17T09:41:13,828 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/c8ebc4424f7749dfb92eefb89df21b1c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c8ebc4424f7749dfb92eefb89df21b1c 2024-11-17T09:41:13,837 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into c8ebc4424f7749dfb92eefb89df21b1c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:13,837 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:13,837 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=13, startTime=1731836473296; duration=0sec 2024-11-17T09:41:13,837 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:13,837 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:13,837 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:13,838 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:13,838 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:13,838 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,840 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/fe4c1f701431461d8832b4f29ccf9007, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/121ebb11f86f4dd5a03001e8a957f924, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/08caa9be016247b880ece16c5f2d0a45] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=36.0 K 2024-11-17T09:41:13,841 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting fe4c1f701431461d8832b4f29ccf9007, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731836471253 2024-11-17T09:41:13,841 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 121ebb11f86f4dd5a03001e8a957f924, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731836471416 2024-11-17T09:41:13,842 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 08caa9be016247b880ece16c5f2d0a45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731836472610 2024-11-17T09:41:13,854 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:13,854 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19119bb619e449ed8cb88be2383fad0c 2024-11-17T09:41:13,855 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c0193b3f092948059ee1a00ad76b4c4d is 50, key is test_row_0/C:col10/1731836472610/Put/seqid=0 2024-11-17T09:41:13,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742029_1205 (size=31583) 2024-11-17T09:41:13,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:13,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:13,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:13,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:13,885 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/9e0f4da92ac94b15b04a418f8740d9f7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/9e0f4da92ac94b15b04a418f8740d9f7 2024-11-17T09:41:13,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/4875dbcd9ead44158026358f6590e207 is 50, key is test_row_0/B:col10/1731836472677/Put/seqid=0 2024-11-17T09:41:13,894 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into 9e0f4da92ac94b15b04a418f8740d9f7(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:13,894 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:13,894 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=13, startTime=1731836473296; duration=0sec 2024-11-17T09:41:13,894 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:13,895 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:13,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742030_1206 (size=12629) 2024-11-17T09:41:13,903 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c0193b3f092948059ee1a00ad76b4c4d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c0193b3f092948059ee1a00ad76b4c4d 2024-11-17T09:41:13,908 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into c0193b3f092948059ee1a00ad76b4c4d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:13,908 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:13,908 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=13, startTime=1731836473296; duration=0sec 2024-11-17T09:41:13,908 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:13,909 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:13,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742031_1207 (size=12251) 2024-11-17T09:41:13,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836533980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836534000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836534000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836534000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:14,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:14,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T09:41:14,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:14,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:14,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/4875dbcd9ead44158026358f6590e207 2024-11-17T09:41:14,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/11789be0ce2f4eb09abbc18ca8ca36eb is 50, key is test_row_0/C:col10/1731836472677/Put/seqid=0 2024-11-17T09:41:14,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742032_1208 (size=12251) 2024-11-17T09:41:14,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:14,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836534490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836534520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836534520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836534520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:14,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/11789be0ce2f4eb09abbc18ca8ca36eb 2024-11-17T09:41:14,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19119bb619e449ed8cb88be2383fad0c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19119bb619e449ed8cb88be2383fad0c 2024-11-17T09:41:14,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19119bb619e449ed8cb88be2383fad0c, entries=150, sequenceid=258, filesize=30.5 K 2024-11-17T09:41:14,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/4875dbcd9ead44158026358f6590e207 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4875dbcd9ead44158026358f6590e207 2024-11-17T09:41:14,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4875dbcd9ead44158026358f6590e207, entries=150, sequenceid=258, filesize=12.0 K 2024-11-17T09:41:14,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/11789be0ce2f4eb09abbc18ca8ca36eb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/11789be0ce2f4eb09abbc18ca8ca36eb 2024-11-17T09:41:14,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/11789be0ce2f4eb09abbc18ca8ca36eb, entries=150, sequenceid=258, filesize=12.0 K 2024-11-17T09:41:14,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 78abf17b9dde42332f107c0d3469140f in 1482ms, sequenceid=258, compaction requested=false 2024-11-17T09:41:14,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:14,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T09:41:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:14,803 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T09:41:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:14,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:14,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:14,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:14,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:14,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173b1c8ba512f54505b58ac122316e3e02_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836473339/Put/seqid=0 2024-11-17T09:41:14,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742033_1209 (size=12454) 2024-11-17T09:41:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:14,895 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173b1c8ba512f54505b58ac122316e3e02_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173b1c8ba512f54505b58ac122316e3e02_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7aafefc1c4904c9dafb8c5eb229b7ac0, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:14,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7aafefc1c4904c9dafb8c5eb229b7ac0 is 175, key is test_row_0/A:col10/1731836473339/Put/seqid=0 2024-11-17T09:41:14,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742034_1210 (size=31255) 2024-11-17T09:41:14,913 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7aafefc1c4904c9dafb8c5eb229b7ac0 2024-11-17T09:41:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/2c4209c3eb754de29ba0ded04352138f is 50, key is test_row_0/B:col10/1731836473339/Put/seqid=0 2024-11-17T09:41:14,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836534954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:14,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742035_1211 (size=12301) 2024-11-17T09:41:15,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836535060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T09:41:15,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836535272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:15,363 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/2c4209c3eb754de29ba0ded04352138f 2024-11-17T09:41:15,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/02edd971d09f414b9441a8995b6017fb is 50, key is test_row_0/C:col10/1731836473339/Put/seqid=0 2024-11-17T09:41:15,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742036_1212 (size=12301) 2024-11-17T09:41:15,402 INFO [master/70e37c475432:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T09:41:15,402 INFO [master/70e37c475432:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T09:41:15,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836535509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:15,513 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:15,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836535540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:15,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836535540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:15,542 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:15,543 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:15,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836535540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:15,547 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836535589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:15,784 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/02edd971d09f414b9441a8995b6017fb 2024-11-17T09:41:15,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7aafefc1c4904c9dafb8c5eb229b7ac0 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7aafefc1c4904c9dafb8c5eb229b7ac0 2024-11-17T09:41:15,794 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7aafefc1c4904c9dafb8c5eb229b7ac0, entries=150, sequenceid=274, filesize=30.5 K 2024-11-17T09:41:15,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/2c4209c3eb754de29ba0ded04352138f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/2c4209c3eb754de29ba0ded04352138f 2024-11-17T09:41:15,802 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/2c4209c3eb754de29ba0ded04352138f, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T09:41:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/02edd971d09f414b9441a8995b6017fb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/02edd971d09f414b9441a8995b6017fb 2024-11-17T09:41:15,812 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/02edd971d09f414b9441a8995b6017fb, entries=150, sequenceid=274, filesize=12.0 K 2024-11-17T09:41:15,814 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 78abf17b9dde42332f107c0d3469140f in 1010ms, sequenceid=274, compaction requested=true 2024-11-17T09:41:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:15,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T09:41:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T09:41:15,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T09:41:15,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8700 sec 2024-11-17T09:41:15,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8780 sec 2024-11-17T09:41:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:16,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:41:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:16,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178ea88b4da4e940eeae339140a798699b_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836476100/Put/seqid=0 2024-11-17T09:41:16,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742037_1213 (size=14994) 2024-11-17T09:41:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836536166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:16,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836536280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:16,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836536490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:16,546 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:16,550 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178ea88b4da4e940eeae339140a798699b_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178ea88b4da4e940eeae339140a798699b_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:16,551 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/a97088bf3a29436b9b665552ebe0f705, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:16,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/a97088bf3a29436b9b665552ebe0f705 is 175, key is test_row_0/A:col10/1731836476100/Put/seqid=0 2024-11-17T09:41:16,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742038_1214 (size=39949) 2024-11-17T09:41:16,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/a97088bf3a29436b9b665552ebe0f705 2024-11-17T09:41:16,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/40e1087d7c0b46b69c8a4d0dfb03073a is 50, key is test_row_0/B:col10/1731836476100/Put/seqid=0 2024-11-17T09:41:16,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742039_1215 (size=12301) 2024-11-17T09:41:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836536809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:16,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/40e1087d7c0b46b69c8a4d0dfb03073a 2024-11-17T09:41:17,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/4195443e2cf14818809f3d025ab8e4dc is 50, key is test_row_0/C:col10/1731836476100/Put/seqid=0 2024-11-17T09:41:17,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742040_1216 (size=12301) 2024-11-17T09:41:17,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/4195443e2cf14818809f3d025ab8e4dc 2024-11-17T09:41:17,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/a97088bf3a29436b9b665552ebe0f705 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/a97088bf3a29436b9b665552ebe0f705 2024-11-17T09:41:17,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/a97088bf3a29436b9b665552ebe0f705, entries=200, sequenceid=294, filesize=39.0 K 2024-11-17T09:41:17,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/40e1087d7c0b46b69c8a4d0dfb03073a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/40e1087d7c0b46b69c8a4d0dfb03073a 2024-11-17T09:41:17,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/40e1087d7c0b46b69c8a4d0dfb03073a, entries=150, sequenceid=294, filesize=12.0 K 2024-11-17T09:41:17,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/4195443e2cf14818809f3d025ab8e4dc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/4195443e2cf14818809f3d025ab8e4dc 2024-11-17T09:41:17,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/4195443e2cf14818809f3d025ab8e4dc, entries=150, sequenceid=294, filesize=12.0 K 2024-11-17T09:41:17,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 78abf17b9dde42332f107c0d3469140f in 962ms, sequenceid=294, compaction requested=true 2024-11-17T09:41:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:17,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:17,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:17,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:17,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:17,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:41:17,066 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:17,066 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:17,067 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49482 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:17,067 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:17,067 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133992 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:17,067 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:17,067 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:17,067 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:17,067 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c8ebc4424f7749dfb92eefb89df21b1c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4875dbcd9ead44158026358f6590e207, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/2c4209c3eb754de29ba0ded04352138f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/40e1087d7c0b46b69c8a4d0dfb03073a] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=48.3 K 2024-11-17T09:41:17,067 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/9e0f4da92ac94b15b04a418f8740d9f7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19119bb619e449ed8cb88be2383fad0c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7aafefc1c4904c9dafb8c5eb229b7ac0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/a97088bf3a29436b9b665552ebe0f705] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=130.9 K 2024-11-17T09:41:17,067 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:17,068 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/9e0f4da92ac94b15b04a418f8740d9f7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19119bb619e449ed8cb88be2383fad0c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7aafefc1c4904c9dafb8c5eb229b7ac0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/a97088bf3a29436b9b665552ebe0f705] 2024-11-17T09:41:17,068 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ebc4424f7749dfb92eefb89df21b1c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731836472610 2024-11-17T09:41:17,068 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0f4da92ac94b15b04a418f8740d9f7, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731836472610 2024-11-17T09:41:17,068 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4875dbcd9ead44158026358f6590e207, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731836472677 2024-11-17T09:41:17,069 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19119bb619e449ed8cb88be2383fad0c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731836472677 2024-11-17T09:41:17,069 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c4209c3eb754de29ba0ded04352138f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731836473320 2024-11-17T09:41:17,069 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aafefc1c4904c9dafb8c5eb229b7ac0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731836473320 2024-11-17T09:41:17,069 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 40e1087d7c0b46b69c8a4d0dfb03073a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731836474931 2024-11-17T09:41:17,069 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97088bf3a29436b9b665552ebe0f705, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731836474931 2024-11-17T09:41:17,080 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T09:41:17,100 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:17,101 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T09:41:17,105 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T09:41:17,108 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:17,109 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/139081fecff446be87a4d478f8f2ea0d is 50, key is test_row_0/B:col10/1731836476100/Put/seqid=0 2024-11-17T09:41:17,112 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:17,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:17,117 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117dfc5a44f69d44307a419a1a20722062b_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:17,120 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117dfc5a44f69d44307a419a1a20722062b_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:17,121 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dfc5a44f69d44307a419a1a20722062b_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:17,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742041_1217 (size=12915) 2024-11-17T09:41:17,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742042_1218 (size=4469) 2024-11-17T09:41:17,182 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#183 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:17,183 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c34a597b329b483393aa085749c1552c is 175, key is test_row_0/A:col10/1731836476100/Put/seqid=0 2024-11-17T09:41:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T09:41:17,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742043_1219 (size=31869) 2024-11-17T09:41:17,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T09:41:17,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:17,266 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T09:41:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:17,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:17,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111715d280a12278400a8616cbee4500dba1_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836476164/Put/seqid=0 2024-11-17T09:41:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742044_1220 (size=12454) 2024-11-17T09:41:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:17,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:17,330 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111715d280a12278400a8616cbee4500dba1_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111715d280a12278400a8616cbee4500dba1_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/17b4a624b61d4c9cb1669d13e1c82869, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:17,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/17b4a624b61d4c9cb1669d13e1c82869 is 175, key is test_row_0/A:col10/1731836476164/Put/seqid=0 2024-11-17T09:41:17,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742045_1221 (size=31255) 2024-11-17T09:41:17,359 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/17b4a624b61d4c9cb1669d13e1c82869 2024-11-17T09:41:17,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e1f4e33339c6404f9586e3ecd3eefa70 is 50, key is test_row_0/B:col10/1731836476164/Put/seqid=0 2024-11-17T09:41:17,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742046_1222 (size=12301) 2024-11-17T09:41:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T09:41:17,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836537453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:17,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836537540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:17,542 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:17,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836537550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:17,556 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836537569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836537569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:17,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836537570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:17,577 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:17,578 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/139081fecff446be87a4d478f8f2ea0d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/139081fecff446be87a4d478f8f2ea0d 2024-11-17T09:41:17,578 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:17,591 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into 139081fecff446be87a4d478f8f2ea0d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:17,591 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:17,591 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=12, startTime=1731836477066; duration=0sec 2024-11-17T09:41:17,591 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:17,591 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:17,591 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:17,601 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49482 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:17,601 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:17,601 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:17,601 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c0193b3f092948059ee1a00ad76b4c4d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/11789be0ce2f4eb09abbc18ca8ca36eb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/02edd971d09f414b9441a8995b6017fb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/4195443e2cf14818809f3d025ab8e4dc] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=48.3 K 2024-11-17T09:41:17,602 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c0193b3f092948059ee1a00ad76b4c4d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731836472610 2024-11-17T09:41:17,602 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 11789be0ce2f4eb09abbc18ca8ca36eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731836472677 2024-11-17T09:41:17,603 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 02edd971d09f414b9441a8995b6017fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731836473320 2024-11-17T09:41:17,603 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4195443e2cf14818809f3d025ab8e4dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731836474931 2024-11-17T09:41:17,618 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/c34a597b329b483393aa085749c1552c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c34a597b329b483393aa085749c1552c 2024-11-17T09:41:17,628 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into c34a597b329b483393aa085749c1552c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:17,628 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:17,628 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=12, startTime=1731836477065; duration=0sec 2024-11-17T09:41:17,628 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#187 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:17,628 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:17,628 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:17,629 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/1c46a7c219bf49bc9c6c1b80d9de3897 is 50, key is test_row_0/C:col10/1731836476100/Put/seqid=0 2024-11-17T09:41:17,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742047_1223 (size=12915) 2024-11-17T09:41:17,675 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/1c46a7c219bf49bc9c6c1b80d9de3897 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1c46a7c219bf49bc9c6c1b80d9de3897 2024-11-17T09:41:17,681 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into 1c46a7c219bf49bc9c6c1b80d9de3897(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:17,681 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:17,681 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=12, startTime=1731836477066; duration=0sec 2024-11-17T09:41:17,682 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:17,682 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T09:41:17,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836537789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:17,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:17,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:17,796 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e1f4e33339c6404f9586e3ecd3eefa70 2024-11-17T09:41:17,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/e9b3c157fe284261b1bd779596acc8fa is 50, key is test_row_0/C:col10/1731836476164/Put/seqid=0 2024-11-17T09:41:17,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742048_1224 (size=12301) 2024-11-17T09:41:17,833 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/e9b3c157fe284261b1bd779596acc8fa 2024-11-17T09:41:17,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/17b4a624b61d4c9cb1669d13e1c82869 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/17b4a624b61d4c9cb1669d13e1c82869 2024-11-17T09:41:17,846 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/17b4a624b61d4c9cb1669d13e1c82869, entries=150, sequenceid=306, filesize=30.5 K 2024-11-17T09:41:17,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/e1f4e33339c6404f9586e3ecd3eefa70 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e1f4e33339c6404f9586e3ecd3eefa70 2024-11-17T09:41:17,853 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e1f4e33339c6404f9586e3ecd3eefa70, entries=150, sequenceid=306, filesize=12.0 K 2024-11-17T09:41:17,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/e9b3c157fe284261b1bd779596acc8fa as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/e9b3c157fe284261b1bd779596acc8fa 2024-11-17T09:41:17,860 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/e9b3c157fe284261b1bd779596acc8fa, entries=150, sequenceid=306, filesize=12.0 K 2024-11-17T09:41:17,861 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 78abf17b9dde42332f107c0d3469140f in 595ms, sequenceid=306, compaction requested=false 2024-11-17T09:41:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T09:41:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T09:41:17,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T09:41:17,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-17T09:41:17,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 765 msec 2024-11-17T09:41:18,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836477429, max timestamp=1731836478103Num uniques 859; 2024-11-17T09:41:18,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:18,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836477429, max timestamp=1731836478103Num uniques 859; 2024-11-17T09:41:18,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836477429, max timestamp=1731836478103Num uniques 859; 2024-11-17T09:41:18,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:18,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:18,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-17T09:41:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:18,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111734f2ff5b28744bd880e808530b202ade_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836478124/Put/seqid=0 2024-11-17T09:41:18,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742049_1225 (size=17534) 2024-11-17T09:41:18,189 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:18,196 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111734f2ff5b28744bd880e808530b202ade_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111734f2ff5b28744bd880e808530b202ade_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:18,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/78106067ec9e4f3090250f3c6ddaeaf1, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:18,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/78106067ec9e4f3090250f3c6ddaeaf1 is 175, key is test_row_0/A:col10/1731836478124/Put/seqid=0 2024-11-17T09:41:18,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742050_1226 (size=48639) 2024-11-17T09:41:18,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/78106067ec9e4f3090250f3c6ddaeaf1 2024-11-17T09:41:18,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/0fcadcc093f14668827bdfce50be1103 is 50, key is test_row_0/B:col10/1731836478124/Put/seqid=0 2024-11-17T09:41:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T09:41:18,241 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:18,241 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T09:41:18,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T09:41:18,245 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:18,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:18,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742051_1227 (size=12301) 2024-11-17T09:41:18,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/0fcadcc093f14668827bdfce50be1103 2024-11-17T09:41:18,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/b6ecf2c3a0884368bc843ac48894f5f3 is 50, key is test_row_0/C:col10/1731836478124/Put/seqid=0 2024-11-17T09:41:18,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836478250, max timestamp=1731836478290Num uniques 859; 2024-11-17T09:41:18,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:18,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836478250, max timestamp=1731836478290Num uniques 859; 2024-11-17T09:41:18,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:18,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836478250, max timestamp=1731836478290Num uniques 859; 2024-11-17T09:41:18,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:18,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742052_1228 (size=12301) 2024-11-17T09:41:18,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/b6ecf2c3a0884368bc843ac48894f5f3 2024-11-17T09:41:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/78106067ec9e4f3090250f3c6ddaeaf1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/78106067ec9e4f3090250f3c6ddaeaf1 2024-11-17T09:41:18,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/78106067ec9e4f3090250f3c6ddaeaf1, entries=250, sequenceid=334, filesize=47.5 K 2024-11-17T09:41:18,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/0fcadcc093f14668827bdfce50be1103 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0fcadcc093f14668827bdfce50be1103 2024-11-17T09:41:18,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0fcadcc093f14668827bdfce50be1103, entries=150, sequenceid=334, filesize=12.0 K 2024-11-17T09:41:18,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/b6ecf2c3a0884368bc843ac48894f5f3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/b6ecf2c3a0884368bc843ac48894f5f3 2024-11-17T09:41:18,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/b6ecf2c3a0884368bc843ac48894f5f3, entries=150, sequenceid=334, filesize=12.0 K 2024-11-17T09:41:18,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T09:41:18,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=60.38 KB/61830 for 78abf17b9dde42332f107c0d3469140f in 224ms, sequenceid=334, compaction requested=true 2024-11-17T09:41:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:18,351 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:41:18,353 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:18,354 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:18,354 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:18,354 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:18,355 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/139081fecff446be87a4d478f8f2ea0d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e1f4e33339c6404f9586e3ecd3eefa70, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0fcadcc093f14668827bdfce50be1103] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=36.6 K 2024-11-17T09:41:18,355 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:18,355 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:18,356 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,356 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c34a597b329b483393aa085749c1552c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/17b4a624b61d4c9cb1669d13e1c82869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/78106067ec9e4f3090250f3c6ddaeaf1] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=109.1 K 2024-11-17T09:41:18,356 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=67.09 KB heapSize=185.77 KB 2024-11-17T09:41:18,356 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c34a597b329b483393aa085749c1552c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/17b4a624b61d4c9cb1669d13e1c82869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/78106067ec9e4f3090250f3c6ddaeaf1] 2024-11-17T09:41:18,356 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 139081fecff446be87a4d478f8f2ea0d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731836474931 2024-11-17T09:41:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:18,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:18,357 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e1f4e33339c6404f9586e3ecd3eefa70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731836476137 2024-11-17T09:41:18,357 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fcadcc093f14668827bdfce50be1103, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731836478111 2024-11-17T09:41:18,357 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c34a597b329b483393aa085749c1552c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731836474931 2024-11-17T09:41:18,358 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b4a624b61d4c9cb1669d13e1c82869, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731836476137 2024-11-17T09:41:18,358 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78106067ec9e4f3090250f3c6ddaeaf1, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731836477429 2024-11-17T09:41:18,375 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:18,376 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/84ae1654f1c74f648393ac77e3699851 is 50, key is test_row_0/B:col10/1731836478124/Put/seqid=0 2024-11-17T09:41:18,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173c054931c88443059f6578831fc5068f_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836478337/Put/seqid=0 2024-11-17T09:41:18,385 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:18,396 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117e1e2ab9c53614de9b6af3ff03c4d8eab_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:18,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,398 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117e1e2ab9c53614de9b6af3ff03c4d8eab_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:18,398 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e1e2ab9c53614de9b6af3ff03c4d8eab_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742054_1230 (size=14994) 2024-11-17T09:41:18,448 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:18,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173c054931c88443059f6578831fc5068f_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173c054931c88443059f6578831fc5068f_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:18,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0292aec2624b494e8e08526e2862a77c, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:18,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0292aec2624b494e8e08526e2862a77c is 175, key is test_row_0/A:col10/1731836478337/Put/seqid=0 2024-11-17T09:41:18,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742053_1229 (size=13017) 2024-11-17T09:41:18,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742055_1231 (size=4469) 2024-11-17T09:41:18,473 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/84ae1654f1c74f648393ac77e3699851 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/84ae1654f1c74f648393ac77e3699851 2024-11-17T09:41:18,492 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into 84ae1654f1c74f648393ac77e3699851(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:18,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742056_1232 (size=39949) 2024-11-17T09:41:18,492 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:18,492 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=13, startTime=1731836478351; duration=0sec 2024-11-17T09:41:18,492 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:18,492 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:18,493 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:18,494 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:18,494 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:18,494 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,494 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1c46a7c219bf49bc9c6c1b80d9de3897, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/e9b3c157fe284261b1bd779596acc8fa, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/b6ecf2c3a0884368bc843ac48894f5f3] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=36.6 K 2024-11-17T09:41:18,495 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c46a7c219bf49bc9c6c1b80d9de3897, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731836474931 2024-11-17T09:41:18,495 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b3c157fe284261b1bd779596acc8fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731836476137 2024-11-17T09:41:18,496 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b6ecf2c3a0884368bc843ac48894f5f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731836478111 2024-11-17T09:41:18,506 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:18,506 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/d31bb494d2504dd7884e599a166f2c26 is 50, key is test_row_0/C:col10/1731836478124/Put/seqid=0 2024-11-17T09:41:18,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836538528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:18,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742057_1233 (size=13017) 2024-11-17T09:41:18,549 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/d31bb494d2504dd7884e599a166f2c26 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d31bb494d2504dd7884e599a166f2c26 2024-11-17T09:41:18,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,558 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into d31bb494d2504dd7884e599a166f2c26(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:18,558 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:18,558 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=13, startTime=1731836478351; duration=0sec 2024-11-17T09:41:18,558 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:18,558 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T09:41:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836538640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:18,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836538850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:18,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:18,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,859 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#194 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:18,860 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/18a699164ba54b0ab3ae3ca904fe0357 is 175, key is test_row_0/A:col10/1731836478124/Put/seqid=0 2024-11-17T09:41:18,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742058_1234 (size=31971) 2024-11-17T09:41:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T09:41:18,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0292aec2624b494e8e08526e2862a77c 2024-11-17T09:41:18,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/6f3f8fb28053480fb6be57e26420c34a is 50, key is test_row_0/B:col10/1731836478337/Put/seqid=0 2024-11-17T09:41:18,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742059_1235 (size=12301) 2024-11-17T09:41:19,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836539159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:19,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,274 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/18a699164ba54b0ab3ae3ca904fe0357 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/18a699164ba54b0ab3ae3ca904fe0357 2024-11-17T09:41:19,279 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into 18a699164ba54b0ab3ae3ca904fe0357(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:19,279 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:19,279 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=13, startTime=1731836478351; duration=0sec 2024-11-17T09:41:19,279 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:19,279 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:19,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/6f3f8fb28053480fb6be57e26420c34a 2024-11-17T09:41:19,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/33d6652401b94a6dbdca8320e0468aa1 is 50, key is test_row_0/C:col10/1731836478337/Put/seqid=0 2024-11-17T09:41:19,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742060_1236 (size=12301) 2024-11-17T09:41:19,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T09:41:19,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:19,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836539670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:19,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:19,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/33d6652401b94a6dbdca8320e0468aa1 2024-11-17T09:41:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/0292aec2624b494e8e08526e2862a77c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0292aec2624b494e8e08526e2862a77c 2024-11-17T09:41:19,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0292aec2624b494e8e08526e2862a77c, entries=200, sequenceid=361, filesize=39.0 K 2024-11-17T09:41:19,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/6f3f8fb28053480fb6be57e26420c34a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/6f3f8fb28053480fb6be57e26420c34a 2024-11-17T09:41:19,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/6f3f8fb28053480fb6be57e26420c34a, entries=150, sequenceid=361, filesize=12.0 K 2024-11-17T09:41:19,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/33d6652401b94a6dbdca8320e0468aa1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/33d6652401b94a6dbdca8320e0468aa1 2024-11-17T09:41:19,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/33d6652401b94a6dbdca8320e0468aa1, entries=150, sequenceid=361, filesize=12.0 K 2024-11-17T09:41:19,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for 78abf17b9dde42332f107c0d3469140f in 1380ms, sequenceid=361, compaction requested=false 2024-11-17T09:41:19,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:19,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T09:41:19,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:19,780 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:41:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:19,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172b9fee0a591242f69966e14cf646fe03_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836478525/Put/seqid=0 2024-11-17T09:41:19,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742061_1237 (size=12454) 2024-11-17T09:41:20,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,199 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172b9fee0a591242f69966e14cf646fe03_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172b9fee0a591242f69966e14cf646fe03_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:20,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/26191aec40fa41109c54fbe2b07a7add, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:20,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/26191aec40fa41109c54fbe2b07a7add is 175, key is test_row_0/A:col10/1731836478525/Put/seqid=0 2024-11-17T09:41:20,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742062_1238 (size=31255) 2024-11-17T09:41:20,210 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/26191aec40fa41109c54fbe2b07a7add 2024-11-17T09:41:20,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/370908dfbfad45ca970079bdb9a68d89 is 50, key is test_row_0/B:col10/1731836478525/Put/seqid=0 2024-11-17T09:41:20,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742063_1239 (size=12301) 2024-11-17T09:41:20,236 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/370908dfbfad45ca970079bdb9a68d89 2024-11-17T09:41:20,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7cc24b4cfc81482cba1ab6321b52ff53 is 50, key is test_row_0/C:col10/1731836478525/Put/seqid=0 2024-11-17T09:41:20,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742064_1240 (size=12301) 2024-11-17T09:41:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T09:41:20,651 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7cc24b4cfc81482cba1ab6321b52ff53 2024-11-17T09:41:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/26191aec40fa41109c54fbe2b07a7add as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/26191aec40fa41109c54fbe2b07a7add 2024-11-17T09:41:20,660 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/26191aec40fa41109c54fbe2b07a7add, entries=150, sequenceid=384, filesize=30.5 K 2024-11-17T09:41:20,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/370908dfbfad45ca970079bdb9a68d89 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/370908dfbfad45ca970079bdb9a68d89 2024-11-17T09:41:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,665 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/370908dfbfad45ca970079bdb9a68d89, entries=150, sequenceid=384, filesize=12.0 K 2024-11-17T09:41:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7cc24b4cfc81482cba1ab6321b52ff53 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7cc24b4cfc81482cba1ab6321b52ff53 2024-11-17T09:41:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,671 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7cc24b4cfc81482cba1ab6321b52ff53, entries=150, sequenceid=384, filesize=12.0 K 2024-11-17T09:41:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,672 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 78abf17b9dde42332f107c0d3469140f in 892ms, sequenceid=384, compaction requested=true 2024-11-17T09:41:20,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:20,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:20,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T09:41:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T09:41:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T09:41:20,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4290 sec 2024-11-17T09:41:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4340 sec 2024-11-17T09:41:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:41:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:20,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a5a84de21d6c4e048094b9d10b40c7ee_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836480762/Put/seqid=0 2024-11-17T09:41:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742065_1241 (size=22618) 2024-11-17T09:41:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,824 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a5a84de21d6c4e048094b9d10b40c7ee_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a5a84de21d6c4e048094b9d10b40c7ee_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:20,827 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19dd7084a8bd4b9a8533deacacff3a7c, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:20,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19dd7084a8bd4b9a8533deacacff3a7c is 175, key is test_row_0/A:col10/1731836480762/Put/seqid=0 2024-11-17T09:41:20,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742066_1242 (size=66023) 2024-11-17T09:41:20,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836480871, max timestamp=1731836480899Num uniques 859; 2024-11-17T09:41:20,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836480871, max timestamp=1731836480899Num uniques 859; 2024-11-17T09:41:20,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836480871, max timestamp=1731836480899Num uniques 859; 2024-11-17T09:41:20,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:20,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:20,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836540968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:20,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836541080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19dd7084a8bd4b9a8533deacacff3a7c 2024-11-17T09:41:21,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/b5d532da7fee4e8b8a86dde42d41e676 is 50, key is test_row_0/B:col10/1731836480762/Put/seqid=0 2024-11-17T09:41:21,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742067_1243 (size=12301) 2024-11-17T09:41:21,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836541289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:21,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731836541559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:21,563 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731836541600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731836541600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836541600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:21,607 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,607 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36270 deadline: 1731836541610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:21,614 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 78abf17b9dde42332f107c0d3469140f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:21,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/b5d532da7fee4e8b8a86dde42d41e676 2024-11-17T09:41:21,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/dbbd12d2aeac4e089352e22d92422eb4 is 50, key is test_row_0/C:col10/1731836480762/Put/seqid=0 2024-11-17T09:41:21,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742068_1244 (size=12301) 2024-11-17T09:41:22,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/dbbd12d2aeac4e089352e22d92422eb4 2024-11-17T09:41:22,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/19dd7084a8bd4b9a8533deacacff3a7c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19dd7084a8bd4b9a8533deacacff3a7c 2024-11-17T09:41:22,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19dd7084a8bd4b9a8533deacacff3a7c, entries=350, sequenceid=394, filesize=64.5 K 2024-11-17T09:41:22,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/b5d532da7fee4e8b8a86dde42d41e676 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/b5d532da7fee4e8b8a86dde42d41e676 2024-11-17T09:41:22,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/b5d532da7fee4e8b8a86dde42d41e676, entries=150, sequenceid=394, filesize=12.0 K 2024-11-17T09:41:22,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/dbbd12d2aeac4e089352e22d92422eb4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/dbbd12d2aeac4e089352e22d92422eb4 2024-11-17T09:41:22,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/dbbd12d2aeac4e089352e22d92422eb4, entries=150, sequenceid=394, filesize=12.0 K 2024-11-17T09:41:22,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 78abf17b9dde42332f107c0d3469140f in 1321ms, sequenceid=394, compaction requested=true 2024-11-17T09:41:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:22,100 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:22,100 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:22,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:22,102 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 169198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:22,102 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:22,102 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,102 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/18a699164ba54b0ab3ae3ca904fe0357, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0292aec2624b494e8e08526e2862a77c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/26191aec40fa41109c54fbe2b07a7add, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19dd7084a8bd4b9a8533deacacff3a7c] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=165.2 K 2024-11-17T09:41:22,102 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,102 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/18a699164ba54b0ab3ae3ca904fe0357, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0292aec2624b494e8e08526e2862a77c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/26191aec40fa41109c54fbe2b07a7add, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19dd7084a8bd4b9a8533deacacff3a7c] 2024-11-17T09:41:22,102 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:22,102 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:22,102 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,103 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/84ae1654f1c74f648393ac77e3699851, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/6f3f8fb28053480fb6be57e26420c34a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/370908dfbfad45ca970079bdb9a68d89, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/b5d532da7fee4e8b8a86dde42d41e676] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=48.8 K 2024-11-17T09:41:22,103 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a699164ba54b0ab3ae3ca904fe0357, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731836478111 2024-11-17T09:41:22,103 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ae1654f1c74f648393ac77e3699851, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731836478111 2024-11-17T09:41:22,103 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3f8fb28053480fb6be57e26420c34a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731836478329 2024-11-17T09:41:22,104 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0292aec2624b494e8e08526e2862a77c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731836478311 2024-11-17T09:41:22,104 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 370908dfbfad45ca970079bdb9a68d89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731836478475 2024-11-17T09:41:22,104 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26191aec40fa41109c54fbe2b07a7add, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731836478475 2024-11-17T09:41:22,105 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d532da7fee4e8b8a86dde42d41e676, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731836480762 2024-11-17T09:41:22,105 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19dd7084a8bd4b9a8533deacacff3a7c, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731836480706 2024-11-17T09:41:22,119 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:22,119 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/c85448f7be1c472982344d79291f7dbb is 50, key is test_row_0/B:col10/1731836480762/Put/seqid=0 2024-11-17T09:41:22,125 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:22,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-17T09:41:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:22,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:22,128 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ba004aa80a10499bb102c7dcadec5b69_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:22,131 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ba004aa80a10499bb102c7dcadec5b69_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:22,131 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ba004aa80a10499bb102c7dcadec5b69_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:22,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742069_1245 (size=13153) 2024-11-17T09:41:22,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742070_1246 (size=4469) 2024-11-17T09:41:22,142 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#205 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:22,142 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/033680c8f6ae4e40991bcc0e1117decd is 175, key is test_row_0/A:col10/1731836480762/Put/seqid=0 2024-11-17T09:41:22,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172a61c3c67a654d498e27524cf887b0ed_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836480966/Put/seqid=0 2024-11-17T09:41:22,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742071_1247 (size=32107) 2024-11-17T09:41:22,155 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/033680c8f6ae4e40991bcc0e1117decd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/033680c8f6ae4e40991bcc0e1117decd 2024-11-17T09:41:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742072_1248 (size=14994) 2024-11-17T09:41:22,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:22,161 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into 033680c8f6ae4e40991bcc0e1117decd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:22,161 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:22,161 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=12, startTime=1731836482100; duration=0sec 2024-11-17T09:41:22,161 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:22,161 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:22,161 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:22,162 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:22,162 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:22,162 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,162 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d31bb494d2504dd7884e599a166f2c26, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/33d6652401b94a6dbdca8320e0468aa1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7cc24b4cfc81482cba1ab6321b52ff53, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/dbbd12d2aeac4e089352e22d92422eb4] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=48.8 K 2024-11-17T09:41:22,164 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d31bb494d2504dd7884e599a166f2c26, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731836478111 2024-11-17T09:41:22,164 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172a61c3c67a654d498e27524cf887b0ed_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172a61c3c67a654d498e27524cf887b0ed_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:22,165 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33d6652401b94a6dbdca8320e0468aa1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731836478329 2024-11-17T09:41:22,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/be403470fb3c4434b27167afb4446a31, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:22,165 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc24b4cfc81482cba1ab6321b52ff53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731836478475 2024-11-17T09:41:22,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/be403470fb3c4434b27167afb4446a31 is 175, key is test_row_0/A:col10/1731836480966/Put/seqid=0 2024-11-17T09:41:22,166 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbbd12d2aeac4e089352e22d92422eb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731836480762 2024-11-17T09:41:22,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742073_1249 (size=39949) 2024-11-17T09:41:22,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=430, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/be403470fb3c4434b27167afb4446a31 2024-11-17T09:41:22,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836542171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:22,178 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:22,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/184530c927c64bc58ce659ff293ea0b6 is 50, key is test_row_0/B:col10/1731836480966/Put/seqid=0 2024-11-17T09:41:22,179 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7c7c94ee5d4e4ad19293950e63aabdc4 is 50, key is test_row_0/C:col10/1731836480762/Put/seqid=0 2024-11-17T09:41:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742074_1250 (size=12301) 2024-11-17T09:41:22,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/184530c927c64bc58ce659ff293ea0b6 2024-11-17T09:41:22,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742075_1251 (size=13153) 2024-11-17T09:41:22,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7d37f67a57e1466796b02a701bb9fe1a is 50, key is test_row_0/C:col10/1731836480966/Put/seqid=0 2024-11-17T09:41:22,197 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7c7c94ee5d4e4ad19293950e63aabdc4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7c7c94ee5d4e4ad19293950e63aabdc4 2024-11-17T09:41:22,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742076_1252 (size=12301) 2024-11-17T09:41:22,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7d37f67a57e1466796b02a701bb9fe1a 2024-11-17T09:41:22,202 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into 7c7c94ee5d4e4ad19293950e63aabdc4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:22,202 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:22,203 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=12, startTime=1731836482100; duration=0sec 2024-11-17T09:41:22,203 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:22,203 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/be403470fb3c4434b27167afb4446a31 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/be403470fb3c4434b27167afb4446a31 2024-11-17T09:41:22,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/be403470fb3c4434b27167afb4446a31, entries=200, sequenceid=430, filesize=39.0 K 2024-11-17T09:41:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/184530c927c64bc58ce659ff293ea0b6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/184530c927c64bc58ce659ff293ea0b6 2024-11-17T09:41:22,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/184530c927c64bc58ce659ff293ea0b6, entries=150, sequenceid=430, filesize=12.0 K 2024-11-17T09:41:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7d37f67a57e1466796b02a701bb9fe1a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7d37f67a57e1466796b02a701bb9fe1a 2024-11-17T09:41:22,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7d37f67a57e1466796b02a701bb9fe1a, entries=150, sequenceid=430, filesize=12.0 K 2024-11-17T09:41:22,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 78abf17b9dde42332f107c0d3469140f in 93ms, sequenceid=430, compaction requested=false 2024-11-17T09:41:22,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:22,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:41:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:22,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f5fa5f8889ef447aa2096318202711dc_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836482161/Put/seqid=0 2024-11-17T09:41:22,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742077_1253 (size=14994) 2024-11-17T09:41:22,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836482343, max timestamp=1731836482360Num uniques 859; 2024-11-17T09:41:22,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836482343, max timestamp=1731836482360Num uniques 859; 2024-11-17T09:41:22,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836482343, max timestamp=1731836482360Num uniques 859; 2024-11-17T09:41:22,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:22,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:22,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T09:41:22,420 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:22,420 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T09:41:22,422 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:22,422 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:22,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836542444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:22,541 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/c85448f7be1c472982344d79291f7dbb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c85448f7be1c472982344d79291f7dbb 2024-11-17T09:41:22,547 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into c85448f7be1c472982344d79291f7dbb(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:22,547 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:22,547 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=12, startTime=1731836482100; duration=0sec 2024-11-17T09:41:22,547 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:22,547 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:22,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836542550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:22,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:22,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:22,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,694 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:22,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f5fa5f8889ef447aa2096318202711dc_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f5fa5f8889ef447aa2096318202711dc_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:22,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7b14a4588115495b92900985d2b6c880, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:22,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7b14a4588115495b92900985d2b6c880 is 175, key is test_row_0/A:col10/1731836482161/Put/seqid=0 2024-11-17T09:41:22,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742078_1254 (size=39949) 2024-11-17T09:41:22,704 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7b14a4588115495b92900985d2b6c880 2024-11-17T09:41:22,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/372ca4799e1f403d9f81f390a2cd4fc7 is 50, key is test_row_0/B:col10/1731836482161/Put/seqid=0 2024-11-17T09:41:22,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742079_1255 (size=12301) 2024-11-17T09:41:22,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:22,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836542759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:22,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:22,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:22,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:22,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:23,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:23,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:23,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36320 deadline: 1731836543070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:41:23,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/372ca4799e1f403d9f81f390a2cd4fc7 2024-11-17T09:41:23,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/8984e11476f54b47bb0d99cf40794fac is 50, key is test_row_0/C:col10/1731836482161/Put/seqid=0 2024-11-17T09:41:23,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742080_1256 (size=12301) 2024-11-17T09:41:23,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:23,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/8984e11476f54b47bb0d99cf40794fac 2024-11-17T09:41:23,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/7b14a4588115495b92900985d2b6c880 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7b14a4588115495b92900985d2b6c880 2024-11-17T09:41:23,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7b14a4588115495b92900985d2b6c880, entries=200, sequenceid=442, filesize=39.0 K 2024-11-17T09:41:23,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/372ca4799e1f403d9f81f390a2cd4fc7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/372ca4799e1f403d9f81f390a2cd4fc7 2024-11-17T09:41:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:23,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/372ca4799e1f403d9f81f390a2cd4fc7, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T09:41:23,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/8984e11476f54b47bb0d99cf40794fac as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8984e11476f54b47bb0d99cf40794fac 2024-11-17T09:41:23,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8984e11476f54b47bb0d99cf40794fac, entries=150, sequenceid=442, filesize=12.0 K 2024-11-17T09:41:23,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 78abf17b9dde42332f107c0d3469140f in 1289ms, sequenceid=442, compaction requested=true 2024-11-17T09:41:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:23,571 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:23,571 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:23,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:23,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78abf17b9dde42332f107c0d3469140f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:23,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:23,572 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:23,573 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/A is initiating minor compaction (all files) 2024-11-17T09:41:23,573 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/A in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,573 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/033680c8f6ae4e40991bcc0e1117decd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/be403470fb3c4434b27167afb4446a31, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7b14a4588115495b92900985d2b6c880] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=109.4 K 2024-11-17T09:41:23,573 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,573 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/033680c8f6ae4e40991bcc0e1117decd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/be403470fb3c4434b27167afb4446a31, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7b14a4588115495b92900985d2b6c880] 2024-11-17T09:41:23,573 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:23,573 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/B is initiating minor compaction (all files) 2024-11-17T09:41:23,573 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/B in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,573 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 033680c8f6ae4e40991bcc0e1117decd, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731836480762 2024-11-17T09:41:23,573 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c85448f7be1c472982344d79291f7dbb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/184530c927c64bc58ce659ff293ea0b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/372ca4799e1f403d9f81f390a2cd4fc7] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=36.9 K 2024-11-17T09:41:23,574 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c85448f7be1c472982344d79291f7dbb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731836480762 2024-11-17T09:41:23,574 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting be403470fb3c4434b27167afb4446a31, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731836480940 2024-11-17T09:41:23,574 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 184530c927c64bc58ce659ff293ea0b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731836480940 2024-11-17T09:41:23,575 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b14a4588115495b92900985d2b6c880, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731836482150 2024-11-17T09:41:23,575 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 372ca4799e1f403d9f81f390a2cd4fc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731836482161 2024-11-17T09:41:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:23,592 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:23,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836482422, max timestamp=1731836483582Num uniques 1018; 2024-11-17T09:41:23,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836482422, max timestamp=1731836483582Num uniques 1018; 2024-11-17T09:41:23,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:41:23,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836482422, max timestamp=1731836483582Num uniques 1018; 2024-11-17T09:41:23,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:41:23,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:41:23,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-17T09:41:23,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:23,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:23,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:23,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:23,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:23,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:23,594 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/58cbfe837ec0455aad5613151223bdec is 50, key is test_row_0/B:col10/1731836482161/Put/seqid=0 2024-11-17T09:41:23,596 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:23,605 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111782bf7e6305c54621a67e2527d8f19058_78abf17b9dde42332f107c0d3469140f store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:23,607 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111782bf7e6305c54621a67e2527d8f19058_78abf17b9dde42332f107c0d3469140f, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:23,608 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111782bf7e6305c54621a67e2527d8f19058_78abf17b9dde42332f107c0d3469140f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:23,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171e2ae2a30589489a9ea3c4d9459f02ac_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836483582/Put/seqid=0 2024-11-17T09:41:23,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742081_1257 (size=13255) 2024-11-17T09:41:23,628 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/58cbfe837ec0455aad5613151223bdec as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/58cbfe837ec0455aad5613151223bdec 2024-11-17T09:41:23,635 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/B of 78abf17b9dde42332f107c0d3469140f into 58cbfe837ec0455aad5613151223bdec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:23,635 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:23,636 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/B, priority=13, startTime=1731836483571; duration=0sec 2024-11-17T09:41:23,636 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:23,636 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:B 2024-11-17T09:41:23,636 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:23,638 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:23,638 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 78abf17b9dde42332f107c0d3469140f/C is initiating minor compaction (all files) 2024-11-17T09:41:23,638 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78abf17b9dde42332f107c0d3469140f/C in TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,638 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7c7c94ee5d4e4ad19293950e63aabdc4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7d37f67a57e1466796b02a701bb9fe1a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8984e11476f54b47bb0d99cf40794fac] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp, totalSize=36.9 K 2024-11-17T09:41:23,639 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7c94ee5d4e4ad19293950e63aabdc4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731836480762 2024-11-17T09:41:23,639 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d37f67a57e1466796b02a701bb9fe1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731836480940 2024-11-17T09:41:23,639 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8984e11476f54b47bb0d99cf40794fac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731836482161 2024-11-17T09:41:23,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:23,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:23,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742083_1259 (size=14994) 2024-11-17T09:41:23,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:23,666 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171e2ae2a30589489a9ea3c4d9459f02ac_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171e2ae2a30589489a9ea3c4d9459f02ac_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:23,671 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:23,672 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/8c015c3c2b1e4445b44bda0f04c0a04a is 50, key is test_row_0/C:col10/1731836482161/Put/seqid=0 2024-11-17T09:41:23,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/47004679debe41cab69251e161c02161, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:23,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/47004679debe41cab69251e161c02161 is 175, key is test_row_0/A:col10/1731836483582/Put/seqid=0 2024-11-17T09:41:23,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742082_1258 (size=4469) 2024-11-17T09:41:23,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742084_1260 (size=39949) 2024-11-17T09:41:23,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742085_1261 (size=13255) 2024-11-17T09:41:23,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836483691, max timestamp=1731836483712Num uniques 859; 2024-11-17T09:41:23,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:23,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836483691, max timestamp=1731836483712Num uniques 859; 2024-11-17T09:41:23,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:23,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836483691, max timestamp=1731836483712Num uniques 859; 2024-11-17T09:41:23,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:23,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836483768, max timestamp=1731836483827Num uniques 1018; 2024-11-17T09:41:23,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T09:41:23,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836483768, max timestamp=1731836483827Num uniques 1018; 2024-11-17T09:41:23,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T09:41:23,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731836483768, max timestamp=1731836483827Num uniques 1018; 2024-11-17T09:41:23,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-17T09:41:23,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731836483895, max timestamp=1731836483919Num uniques 1077; 2024-11-17T09:41:23,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-17T09:41:23,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731836483895, max timestamp=1731836483919Num uniques 1077; 2024-11-17T09:41:23,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-17T09:41:23,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731836483895, max timestamp=1731836483919Num uniques 1009; 2024-11-17T09:41:23,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-17T09:41:23,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:23,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731836483974, max timestamp=1731836484000Num uniques 1086; 2024-11-17T09:41:24,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T09:41:24,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731836483974, max timestamp=1731836484000Num uniques 1086; 2024-11-17T09:41:24,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T09:41:24,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731836483974, max timestamp=1731836484000Num uniques 1086; 2024-11-17T09:41:24,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T09:41:24,076 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78abf17b9dde42332f107c0d3469140f#A#compaction#214 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:24,077 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/76e0fb00f9634afeae49f7c0f9b966fe is 175, key is test_row_0/A:col10/1731836482161/Put/seqid=0 2024-11-17T09:41:24,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742086_1262 (size=32209) 2024-11-17T09:41:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731836484066, max timestamp=1731836484090Num uniques 1095; 2024-11-17T09:41:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-17T09:41:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731836484066, max timestamp=1731836484090Num uniques 1095; 2024-11-17T09:41:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-17T09:41:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731836484066, max timestamp=1731836484090Num uniques 1095; 2024-11-17T09:41:24,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-17T09:41:24,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=481, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/47004679debe41cab69251e161c02161 2024-11-17T09:41:24,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/0912dad382f847689717ba657573a8f9 is 50, key is test_row_0/B:col10/1731836483582/Put/seqid=0 2024-11-17T09:41:24,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742087_1263 (size=12301) 2024-11-17T09:41:24,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,115 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/8c015c3c2b1e4445b44bda0f04c0a04a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8c015c3c2b1e4445b44bda0f04c0a04a 2024-11-17T09:41:24,119 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/C of 78abf17b9dde42332f107c0d3469140f into 8c015c3c2b1e4445b44bda0f04c0a04a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:24,119 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:24,119 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/C, priority=13, startTime=1731836483572; duration=0sec 2024-11-17T09:41:24,120 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:24,120 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:C 2024-11-17T09:41:24,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731836484157, max timestamp=1731836484185Num uniques 1054; 2024-11-17T09:41:24,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-17T09:41:24,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731836484157, max timestamp=1731836484185Num uniques 1054; 2024-11-17T09:41:24,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-17T09:41:24,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731836484157, max timestamp=1731836484185Num uniques 1054; 2024-11-17T09:41:24,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-17T09:41:24,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:24,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836484242, max timestamp=1731836484273Num uniques 1063; 2024-11-17T09:41:24,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836484242, max timestamp=1731836484273Num uniques 1063; 2024-11-17T09:41:24,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-17T09:41:24,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836484242, max timestamp=1731836484273Num uniques 1063; 2024-11-17T09:41:24,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-17T09:41:24,278 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-17T09:41:24,318 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-17T09:41:24,318 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:24,318 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,318 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:24,319 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-17T09:41:24,319 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:24,319 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,319 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:24,319 INFO [Thread-724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-724. 2024-11-17T09:41:24,320 DEBUG [Thread-724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:24,320 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,320 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:24,323 INFO [Thread-718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-718. 2024-11-17T09:41:24,323 DEBUG [Thread-718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:24,323 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,323 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:24,324 INFO [Thread-722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-722. 2024-11-17T09:41:24,324 DEBUG [Thread-722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:24,324 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,324 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:24,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:24,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,486 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/76e0fb00f9634afeae49f7c0f9b966fe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/76e0fb00f9634afeae49f7c0f9b966fe 2024-11-17T09:41:24,491 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78abf17b9dde42332f107c0d3469140f/A of 78abf17b9dde42332f107c0d3469140f into 76e0fb00f9634afeae49f7c0f9b966fe(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:24,491 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:24,491 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., storeName=78abf17b9dde42332f107c0d3469140f/A, priority=13, startTime=1731836483571; duration=0sec 2024-11-17T09:41:24,491 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:24,491 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78abf17b9dde42332f107c0d3469140f:A 2024-11-17T09:41:24,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/0912dad382f847689717ba657573a8f9 2024-11-17T09:41:24,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/12b8986f4a4842529a683f1a394ae614 is 50, key is test_row_0/C:col10/1731836483582/Put/seqid=0 2024-11-17T09:41:24,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742088_1264 (size=12301) 2024-11-17T09:41:24,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:24,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:24,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:24,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. as already flushing 2024-11-17T09:41:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:24,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:24,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/12b8986f4a4842529a683f1a394ae614 2024-11-17T09:41:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/47004679debe41cab69251e161c02161 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/47004679debe41cab69251e161c02161 2024-11-17T09:41:24,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/47004679debe41cab69251e161c02161, entries=200, sequenceid=481, filesize=39.0 K 2024-11-17T09:41:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/0912dad382f847689717ba657573a8f9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0912dad382f847689717ba657573a8f9 2024-11-17T09:41:24,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0912dad382f847689717ba657573a8f9, entries=150, sequenceid=481, filesize=12.0 K 2024-11-17T09:41:24,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/12b8986f4a4842529a683f1a394ae614 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/12b8986f4a4842529a683f1a394ae614 2024-11-17T09:41:24,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/12b8986f4a4842529a683f1a394ae614, entries=150, sequenceid=481, filesize=12.0 K 2024-11-17T09:41:24,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=100.63 KB/103050 for 78abf17b9dde42332f107c0d3469140f in 1337ms, sequenceid=481, compaction requested=false 2024-11-17T09:41:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:25,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T09:41:25,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:25,026 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=100.63 KB heapSize=280.31 KB 2024-11-17T09:41:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:25,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f9e6ef496d004d45990646f7b1efbe58_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_0/A:col10/1731836484309/Put/seqid=0 2024-11-17T09:41:25,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742089_1265 (size=12454) 2024-11-17T09:41:25,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:25,440 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f9e6ef496d004d45990646f7b1efbe58_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f9e6ef496d004d45990646f7b1efbe58_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/8ede896323ff4ee28963f7df7a2de4e3, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/8ede896323ff4ee28963f7df7a2de4e3 is 175, key is test_row_0/A:col10/1731836484309/Put/seqid=0 2024-11-17T09:41:25,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742090_1266 (size=31255) 2024-11-17T09:41:25,848 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=618, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/8ede896323ff4ee28963f7df7a2de4e3 2024-11-17T09:41:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/50b3a962628e447bbfbac07407a9fca8 is 50, key is test_row_0/B:col10/1731836484309/Put/seqid=0 2024-11-17T09:41:25,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742091_1267 (size=12301) 2024-11-17T09:41:26,259 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/50b3a962628e447bbfbac07407a9fca8 2024-11-17T09:41:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c8dd418684874f4b983b1723656fef95 is 50, key is test_row_0/C:col10/1731836484309/Put/seqid=0 2024-11-17T09:41:26,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742092_1268 (size=12301) 2024-11-17T09:41:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:26,670 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c8dd418684874f4b983b1723656fef95 2024-11-17T09:41:26,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/8ede896323ff4ee28963f7df7a2de4e3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/8ede896323ff4ee28963f7df7a2de4e3 2024-11-17T09:41:26,677 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/8ede896323ff4ee28963f7df7a2de4e3, entries=150, sequenceid=618, filesize=30.5 K 2024-11-17T09:41:26,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/50b3a962628e447bbfbac07407a9fca8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/50b3a962628e447bbfbac07407a9fca8 2024-11-17T09:41:26,681 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/50b3a962628e447bbfbac07407a9fca8, entries=150, sequenceid=618, filesize=12.0 K 2024-11-17T09:41:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/c8dd418684874f4b983b1723656fef95 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c8dd418684874f4b983b1723656fef95 2024-11-17T09:41:26,684 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c8dd418684874f4b983b1723656fef95, entries=150, sequenceid=618, filesize=12.0 K 2024-11-17T09:41:26,685 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~280.27 KB/286992, currentSize=0 B/0 for 78abf17b9dde42332f107c0d3469140f in 1659ms, sequenceid=618, compaction requested=true 2024-11-17T09:41:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 78abf17b9dde42332f107c0d3469140f: 2024-11-17T09:41:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T09:41:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T09:41:26,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T09:41:26,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2640 sec 2024-11-17T09:41:26,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 4.2680 sec 2024-11-17T09:41:30,490 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T09:41:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T09:41:30,601 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:31,621 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-17T09:41:31,621 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:31,621 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,621 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:31,631 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-17T09:41:31,631 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:31,631 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,631 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:31,651 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-11-17T09:41:31,651 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:31,651 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,651 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:31,671 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-11-17T09:41:31,671 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:31,671 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,671 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 372 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2371 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2337 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1038 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3113 rows 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1040 2024-11-17T09:41:31,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3118 rows 2024-11-17T09:41:31,672 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:41:31,672 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:41:31,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:31,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:31,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T09:41:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T09:41:31,675 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836491675"}]},"ts":"1731836491675"} 2024-11-17T09:41:31,677 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T09:41:31,677 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T09:41:31,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:41:31,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, UNASSIGN}] 2024-11-17T09:41:31,679 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, UNASSIGN 2024-11-17T09:41:31,679 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:31,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, UNASSIGN because future has completed 2024-11-17T09:41:31,681 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:41:31,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:41:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T09:41:31,833 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:31,833 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:41:31,833 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 78abf17b9dde42332f107c0d3469140f, disabling compactions & flushes 2024-11-17T09:41:31,833 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:31,833 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:31,833 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. after waiting 0 ms 2024-11-17T09:41:31,833 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:31,834 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 78abf17b9dde42332f107c0d3469140f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T09:41:31,834 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=A 2024-11-17T09:41:31,834 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:31,834 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=B 2024-11-17T09:41:31,834 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:31,834 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 78abf17b9dde42332f107c0d3469140f, store=C 2024-11-17T09:41:31,834 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:31,840 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117404123b99b6c402d9f7d8e16689d7b19_78abf17b9dde42332f107c0d3469140f is 50, key is test_row_1/A:col10/1731836491630/Put/seqid=0 2024-11-17T09:41:31,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742093_1269 (size=9914) 2024-11-17T09:41:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T09:41:32,245 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:32,249 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117404123b99b6c402d9f7d8e16689d7b19_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117404123b99b6c402d9f7d8e16689d7b19_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:32,250 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/51b72ec41e944281bf483193cac0d30e, store: [table=TestAcidGuarantees family=A region=78abf17b9dde42332f107c0d3469140f] 2024-11-17T09:41:32,250 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/51b72ec41e944281bf483193cac0d30e is 175, key is test_row_1/A:col10/1731836491630/Put/seqid=0 2024-11-17T09:41:32,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742094_1270 (size=22561) 2024-11-17T09:41:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T09:41:32,655 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=625, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/51b72ec41e944281bf483193cac0d30e 2024-11-17T09:41:32,661 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/12ac92868f2c4332acf2aa0ea1058798 is 50, key is test_row_1/B:col10/1731836491630/Put/seqid=0 2024-11-17T09:41:32,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742095_1271 (size=9857) 2024-11-17T09:41:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T09:41:33,065 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/12ac92868f2c4332acf2aa0ea1058798 2024-11-17T09:41:33,072 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7e6e3c77829c47168c923217880ae0ef is 50, key is test_row_1/C:col10/1731836491630/Put/seqid=0 2024-11-17T09:41:33,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742096_1272 (size=9857) 2024-11-17T09:41:33,476 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=625 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7e6e3c77829c47168c923217880ae0ef 2024-11-17T09:41:33,480 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/A/51b72ec41e944281bf483193cac0d30e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/51b72ec41e944281bf483193cac0d30e 2024-11-17T09:41:33,484 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/51b72ec41e944281bf483193cac0d30e, entries=100, sequenceid=625, filesize=22.0 K 2024-11-17T09:41:33,485 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/B/12ac92868f2c4332acf2aa0ea1058798 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/12ac92868f2c4332acf2aa0ea1058798 2024-11-17T09:41:33,488 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/12ac92868f2c4332acf2aa0ea1058798, entries=100, sequenceid=625, filesize=9.6 K 2024-11-17T09:41:33,488 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/.tmp/C/7e6e3c77829c47168c923217880ae0ef as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7e6e3c77829c47168c923217880ae0ef 2024-11-17T09:41:33,492 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7e6e3c77829c47168c923217880ae0ef, entries=100, sequenceid=625, filesize=9.6 K 2024-11-17T09:41:33,492 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 78abf17b9dde42332f107c0d3469140f in 1659ms, sequenceid=625, compaction requested=true 2024-11-17T09:41:33,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/dd44eb9e67e74118a58a36ebc0bd46fc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0d0ce32f8be14a92adcfe30b56e6f553, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/d025896a0cf64fdbbb9759f2dc3d1a32, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b2a29a88906a4c88ad91c1d2ce2c5a63, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0cef4a08c46340efb6feb2aec26b35a1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/faddbbbd37df47dfb9cbbc9cda781fbd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/ed70c255906b464899c2384183b49632, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c45b594b3ddb48018205be4836267226, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c99d6ab9241f420abdbe6b5bbf02ca78, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b04909572dbe4f649fbe801ce1f13dca, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/6aece789623041dd855eff99054770e4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/f6b321e0c56f4d6189db4750ca57d99f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/e603478be9aa4bf5a9d54d42f220eb96, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/3613cb0db4f74133bac4105354ae50c3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/9e0f4da92ac94b15b04a418f8740d9f7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19119bb619e449ed8cb88be2383fad0c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7aafefc1c4904c9dafb8c5eb229b7ac0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/a97088bf3a29436b9b665552ebe0f705, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c34a597b329b483393aa085749c1552c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/17b4a624b61d4c9cb1669d13e1c82869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/78106067ec9e4f3090250f3c6ddaeaf1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/18a699164ba54b0ab3ae3ca904fe0357, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0292aec2624b494e8e08526e2862a77c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/26191aec40fa41109c54fbe2b07a7add, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19dd7084a8bd4b9a8533deacacff3a7c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/033680c8f6ae4e40991bcc0e1117decd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/be403470fb3c4434b27167afb4446a31, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7b14a4588115495b92900985d2b6c880] to archive 2024-11-17T09:41:33,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:41:33,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/dd44eb9e67e74118a58a36ebc0bd46fc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/dd44eb9e67e74118a58a36ebc0bd46fc 2024-11-17T09:41:33,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0d0ce32f8be14a92adcfe30b56e6f553 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0d0ce32f8be14a92adcfe30b56e6f553 2024-11-17T09:41:33,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/d025896a0cf64fdbbb9759f2dc3d1a32 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/d025896a0cf64fdbbb9759f2dc3d1a32 2024-11-17T09:41:33,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b2a29a88906a4c88ad91c1d2ce2c5a63 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b2a29a88906a4c88ad91c1d2ce2c5a63 2024-11-17T09:41:33,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0cef4a08c46340efb6feb2aec26b35a1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0cef4a08c46340efb6feb2aec26b35a1 2024-11-17T09:41:33,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/faddbbbd37df47dfb9cbbc9cda781fbd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/faddbbbd37df47dfb9cbbc9cda781fbd 2024-11-17T09:41:33,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/ed70c255906b464899c2384183b49632 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/ed70c255906b464899c2384183b49632 2024-11-17T09:41:33,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c45b594b3ddb48018205be4836267226 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c45b594b3ddb48018205be4836267226 2024-11-17T09:41:33,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c99d6ab9241f420abdbe6b5bbf02ca78 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c99d6ab9241f420abdbe6b5bbf02ca78 2024-11-17T09:41:33,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b04909572dbe4f649fbe801ce1f13dca to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/b04909572dbe4f649fbe801ce1f13dca 2024-11-17T09:41:33,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/6aece789623041dd855eff99054770e4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/6aece789623041dd855eff99054770e4 2024-11-17T09:41:33,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/f6b321e0c56f4d6189db4750ca57d99f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/f6b321e0c56f4d6189db4750ca57d99f 2024-11-17T09:41:33,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/e603478be9aa4bf5a9d54d42f220eb96 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/e603478be9aa4bf5a9d54d42f220eb96 2024-11-17T09:41:33,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/3613cb0db4f74133bac4105354ae50c3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/3613cb0db4f74133bac4105354ae50c3 2024-11-17T09:41:33,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/9e0f4da92ac94b15b04a418f8740d9f7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/9e0f4da92ac94b15b04a418f8740d9f7 2024-11-17T09:41:33,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19119bb619e449ed8cb88be2383fad0c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19119bb619e449ed8cb88be2383fad0c 2024-11-17T09:41:33,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7aafefc1c4904c9dafb8c5eb229b7ac0 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7aafefc1c4904c9dafb8c5eb229b7ac0 2024-11-17T09:41:33,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/a97088bf3a29436b9b665552ebe0f705 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/a97088bf3a29436b9b665552ebe0f705 2024-11-17T09:41:33,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c34a597b329b483393aa085749c1552c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/c34a597b329b483393aa085749c1552c 2024-11-17T09:41:33,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/17b4a624b61d4c9cb1669d13e1c82869 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/17b4a624b61d4c9cb1669d13e1c82869 2024-11-17T09:41:33,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/78106067ec9e4f3090250f3c6ddaeaf1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/78106067ec9e4f3090250f3c6ddaeaf1 2024-11-17T09:41:33,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/18a699164ba54b0ab3ae3ca904fe0357 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/18a699164ba54b0ab3ae3ca904fe0357 2024-11-17T09:41:33,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0292aec2624b494e8e08526e2862a77c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/0292aec2624b494e8e08526e2862a77c 2024-11-17T09:41:33,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/26191aec40fa41109c54fbe2b07a7add to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/26191aec40fa41109c54fbe2b07a7add 2024-11-17T09:41:33,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19dd7084a8bd4b9a8533deacacff3a7c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/19dd7084a8bd4b9a8533deacacff3a7c 2024-11-17T09:41:33,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/033680c8f6ae4e40991bcc0e1117decd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/033680c8f6ae4e40991bcc0e1117decd 2024-11-17T09:41:33,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/be403470fb3c4434b27167afb4446a31 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/be403470fb3c4434b27167afb4446a31 2024-11-17T09:41:33,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7b14a4588115495b92900985d2b6c880 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/7b14a4588115495b92900985d2b6c880 2024-11-17T09:41:33,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/01475e5f4cdb402cb528be44c4dd5e89, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/830b261e970a4779ae27657049b6e4a3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e05fd011f97f4ac4b4c0fd84af51e20a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4c44670137d14fe4bfed210c13399776, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/97cfa73cc30a4015b7a0ce9bf6b819e0, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/ce272bed9f1a423db049e8e14ca2b0b1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a5ab610edeb845a1870816ab224f0f3a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d62fbe6e57e14b1e9fe40e2e762c9bdf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a63329cae8c84fd481a8aac2512196f7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/874e58042b224c73a1aef6ad4a3a9610, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e479f5f6ae844cdb8cfd1de590b4cd3d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/1754a5ff0174443cbe1d0d639fadc282, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/942d8d7ead34424eb94b30b556742c5e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c8ebc4424f7749dfb92eefb89df21b1c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d8cc0bd88d3d4bc4895df7cd33dc72df, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4875dbcd9ead44158026358f6590e207, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/2c4209c3eb754de29ba0ded04352138f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/139081fecff446be87a4d478f8f2ea0d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/40e1087d7c0b46b69c8a4d0dfb03073a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e1f4e33339c6404f9586e3ecd3eefa70, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/84ae1654f1c74f648393ac77e3699851, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0fcadcc093f14668827bdfce50be1103, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/6f3f8fb28053480fb6be57e26420c34a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/370908dfbfad45ca970079bdb9a68d89, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c85448f7be1c472982344d79291f7dbb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/b5d532da7fee4e8b8a86dde42d41e676, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/184530c927c64bc58ce659ff293ea0b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/372ca4799e1f403d9f81f390a2cd4fc7] to archive 2024-11-17T09:41:33,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:41:33,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/01475e5f4cdb402cb528be44c4dd5e89 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/01475e5f4cdb402cb528be44c4dd5e89 2024-11-17T09:41:33,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/830b261e970a4779ae27657049b6e4a3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/830b261e970a4779ae27657049b6e4a3 2024-11-17T09:41:33,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e05fd011f97f4ac4b4c0fd84af51e20a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e05fd011f97f4ac4b4c0fd84af51e20a 2024-11-17T09:41:33,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4c44670137d14fe4bfed210c13399776 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4c44670137d14fe4bfed210c13399776 2024-11-17T09:41:33,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/97cfa73cc30a4015b7a0ce9bf6b819e0 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/97cfa73cc30a4015b7a0ce9bf6b819e0 2024-11-17T09:41:33,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/ce272bed9f1a423db049e8e14ca2b0b1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/ce272bed9f1a423db049e8e14ca2b0b1 2024-11-17T09:41:33,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a5ab610edeb845a1870816ab224f0f3a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a5ab610edeb845a1870816ab224f0f3a 2024-11-17T09:41:33,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d62fbe6e57e14b1e9fe40e2e762c9bdf to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d62fbe6e57e14b1e9fe40e2e762c9bdf 2024-11-17T09:41:33,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a63329cae8c84fd481a8aac2512196f7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/a63329cae8c84fd481a8aac2512196f7 2024-11-17T09:41:33,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/874e58042b224c73a1aef6ad4a3a9610 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/874e58042b224c73a1aef6ad4a3a9610 2024-11-17T09:41:33,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e479f5f6ae844cdb8cfd1de590b4cd3d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e479f5f6ae844cdb8cfd1de590b4cd3d 2024-11-17T09:41:33,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/1754a5ff0174443cbe1d0d639fadc282 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/1754a5ff0174443cbe1d0d639fadc282 2024-11-17T09:41:33,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/942d8d7ead34424eb94b30b556742c5e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/942d8d7ead34424eb94b30b556742c5e 2024-11-17T09:41:33,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c8ebc4424f7749dfb92eefb89df21b1c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c8ebc4424f7749dfb92eefb89df21b1c 2024-11-17T09:41:33,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d8cc0bd88d3d4bc4895df7cd33dc72df to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/d8cc0bd88d3d4bc4895df7cd33dc72df 2024-11-17T09:41:33,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4875dbcd9ead44158026358f6590e207 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/4875dbcd9ead44158026358f6590e207 2024-11-17T09:41:33,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/2c4209c3eb754de29ba0ded04352138f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/2c4209c3eb754de29ba0ded04352138f 2024-11-17T09:41:33,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/139081fecff446be87a4d478f8f2ea0d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/139081fecff446be87a4d478f8f2ea0d 2024-11-17T09:41:33,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/40e1087d7c0b46b69c8a4d0dfb03073a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/40e1087d7c0b46b69c8a4d0dfb03073a 2024-11-17T09:41:33,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e1f4e33339c6404f9586e3ecd3eefa70 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/e1f4e33339c6404f9586e3ecd3eefa70 2024-11-17T09:41:33,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/84ae1654f1c74f648393ac77e3699851 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/84ae1654f1c74f648393ac77e3699851 2024-11-17T09:41:33,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0fcadcc093f14668827bdfce50be1103 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0fcadcc093f14668827bdfce50be1103 2024-11-17T09:41:33,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/6f3f8fb28053480fb6be57e26420c34a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/6f3f8fb28053480fb6be57e26420c34a 2024-11-17T09:41:33,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/370908dfbfad45ca970079bdb9a68d89 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/370908dfbfad45ca970079bdb9a68d89 2024-11-17T09:41:33,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c85448f7be1c472982344d79291f7dbb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/c85448f7be1c472982344d79291f7dbb 2024-11-17T09:41:33,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/b5d532da7fee4e8b8a86dde42d41e676 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/b5d532da7fee4e8b8a86dde42d41e676 2024-11-17T09:41:33,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/184530c927c64bc58ce659ff293ea0b6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/184530c927c64bc58ce659ff293ea0b6 2024-11-17T09:41:33,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/372ca4799e1f403d9f81f390a2cd4fc7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/372ca4799e1f403d9f81f390a2cd4fc7 2024-11-17T09:41:33,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7663613b5d8e47eea18d79ef4ee10045, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/2966bdca11e948bebed671230279ecc7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/0c9f8ad027f64fd7a892642a6de7ddf1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/425c2c6acb034905a868271758c3ff44, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d27b146564284b6399650ab746c054ae, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c736bfabc80147d4b04a299fc0a65869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/3d128dcec5654828947b64a9ecbbe491, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/30998ba8db1041e7ada8992ad1fcb234, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1ef639077bdf40878cfc5d5a0b62d92c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/6383867b0828405cad5da18af759b480, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/fe4c1f701431461d8832b4f29ccf9007, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/24202ad268844db69d788d45b88b70f9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/121ebb11f86f4dd5a03001e8a957f924, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c0193b3f092948059ee1a00ad76b4c4d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/08caa9be016247b880ece16c5f2d0a45, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/11789be0ce2f4eb09abbc18ca8ca36eb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/02edd971d09f414b9441a8995b6017fb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1c46a7c219bf49bc9c6c1b80d9de3897, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/4195443e2cf14818809f3d025ab8e4dc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/e9b3c157fe284261b1bd779596acc8fa, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d31bb494d2504dd7884e599a166f2c26, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/b6ecf2c3a0884368bc843ac48894f5f3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/33d6652401b94a6dbdca8320e0468aa1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7cc24b4cfc81482cba1ab6321b52ff53, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7c7c94ee5d4e4ad19293950e63aabdc4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/dbbd12d2aeac4e089352e22d92422eb4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7d37f67a57e1466796b02a701bb9fe1a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8984e11476f54b47bb0d99cf40794fac] to archive 2024-11-17T09:41:33,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:41:33,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7663613b5d8e47eea18d79ef4ee10045 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7663613b5d8e47eea18d79ef4ee10045 2024-11-17T09:41:33,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/2966bdca11e948bebed671230279ecc7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/2966bdca11e948bebed671230279ecc7 2024-11-17T09:41:33,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/0c9f8ad027f64fd7a892642a6de7ddf1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/0c9f8ad027f64fd7a892642a6de7ddf1 2024-11-17T09:41:33,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/425c2c6acb034905a868271758c3ff44 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/425c2c6acb034905a868271758c3ff44 2024-11-17T09:41:33,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d27b146564284b6399650ab746c054ae to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d27b146564284b6399650ab746c054ae 2024-11-17T09:41:33,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c736bfabc80147d4b04a299fc0a65869 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c736bfabc80147d4b04a299fc0a65869 2024-11-17T09:41:33,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/3d128dcec5654828947b64a9ecbbe491 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/3d128dcec5654828947b64a9ecbbe491 2024-11-17T09:41:33,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/30998ba8db1041e7ada8992ad1fcb234 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/30998ba8db1041e7ada8992ad1fcb234 2024-11-17T09:41:33,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1ef639077bdf40878cfc5d5a0b62d92c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1ef639077bdf40878cfc5d5a0b62d92c 2024-11-17T09:41:33,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/6383867b0828405cad5da18af759b480 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/6383867b0828405cad5da18af759b480 2024-11-17T09:41:33,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/fe4c1f701431461d8832b4f29ccf9007 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/fe4c1f701431461d8832b4f29ccf9007 2024-11-17T09:41:33,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/24202ad268844db69d788d45b88b70f9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/24202ad268844db69d788d45b88b70f9 2024-11-17T09:41:33,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/121ebb11f86f4dd5a03001e8a957f924 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/121ebb11f86f4dd5a03001e8a957f924 2024-11-17T09:41:33,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c0193b3f092948059ee1a00ad76b4c4d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c0193b3f092948059ee1a00ad76b4c4d 2024-11-17T09:41:33,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/08caa9be016247b880ece16c5f2d0a45 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/08caa9be016247b880ece16c5f2d0a45 2024-11-17T09:41:33,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/11789be0ce2f4eb09abbc18ca8ca36eb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/11789be0ce2f4eb09abbc18ca8ca36eb 2024-11-17T09:41:33,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/02edd971d09f414b9441a8995b6017fb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/02edd971d09f414b9441a8995b6017fb 2024-11-17T09:41:33,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1c46a7c219bf49bc9c6c1b80d9de3897 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/1c46a7c219bf49bc9c6c1b80d9de3897 2024-11-17T09:41:33,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/4195443e2cf14818809f3d025ab8e4dc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/4195443e2cf14818809f3d025ab8e4dc 2024-11-17T09:41:33,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/e9b3c157fe284261b1bd779596acc8fa to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/e9b3c157fe284261b1bd779596acc8fa 2024-11-17T09:41:33,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d31bb494d2504dd7884e599a166f2c26 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/d31bb494d2504dd7884e599a166f2c26 2024-11-17T09:41:33,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/b6ecf2c3a0884368bc843ac48894f5f3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/b6ecf2c3a0884368bc843ac48894f5f3 2024-11-17T09:41:33,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/33d6652401b94a6dbdca8320e0468aa1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/33d6652401b94a6dbdca8320e0468aa1 2024-11-17T09:41:33,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7cc24b4cfc81482cba1ab6321b52ff53 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7cc24b4cfc81482cba1ab6321b52ff53 2024-11-17T09:41:33,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7c7c94ee5d4e4ad19293950e63aabdc4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7c7c94ee5d4e4ad19293950e63aabdc4 2024-11-17T09:41:33,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/dbbd12d2aeac4e089352e22d92422eb4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/dbbd12d2aeac4e089352e22d92422eb4 2024-11-17T09:41:33,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7d37f67a57e1466796b02a701bb9fe1a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7d37f67a57e1466796b02a701bb9fe1a 2024-11-17T09:41:33,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8984e11476f54b47bb0d99cf40794fac to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8984e11476f54b47bb0d99cf40794fac 2024-11-17T09:41:33,580 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/recovered.edits/628.seqid, newMaxSeqId=628, maxSeqId=4 2024-11-17T09:41:33,581 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. 2024-11-17T09:41:33,581 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 78abf17b9dde42332f107c0d3469140f: Waiting for close lock at 1731836491833Running coprocessor pre-close hooks at 1731836491833Disabling compacts and flushes for region at 1731836491833Disabling writes for close at 1731836491833Obtaining lock to block concurrent updates at 1731836491834 (+1 ms)Preparing flush snapshotting stores in 78abf17b9dde42332f107c0d3469140f at 1731836491834Finished memstore snapshotting TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731836491834Flushing stores of TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f. at 1731836491834Flushing 78abf17b9dde42332f107c0d3469140f/A: creating writer at 1731836491834Flushing mob file 78abf17b9dde42332f107c0d3469140f/A: appending metadata at 1731836491840 (+6 ms)Flushing mob file 78abf17b9dde42332f107c0d3469140f/A: closing flushed file at 1731836491840Flushing 78abf17b9dde42332f107c0d3469140f/A: appending metadata at 1731836492250 (+410 ms)Flushing 78abf17b9dde42332f107c0d3469140f/A: closing flushed file at 1731836492250Flushing 78abf17b9dde42332f107c0d3469140f/B: creating writer at 1731836492658 (+408 ms)Flushing 78abf17b9dde42332f107c0d3469140f/B: appending metadata at 1731836492661 (+3 ms)Flushing 78abf17b9dde42332f107c0d3469140f/B: closing flushed file at 1731836492661Flushing 78abf17b9dde42332f107c0d3469140f/C: creating writer at 1731836493069 (+408 ms)Flushing 78abf17b9dde42332f107c0d3469140f/C: appending metadata at 1731836493071 (+2 ms)Flushing 78abf17b9dde42332f107c0d3469140f/C: closing flushed file at 1731836493071Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1902063e: reopening flushed file at 1731836493480 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e08be3e: reopening flushed file at 1731836493484 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@752e0657: reopening flushed file at 1731836493488 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 78abf17b9dde42332f107c0d3469140f in 1659ms, sequenceid=625, compaction requested=true at 1731836493492 (+4 ms)Writing region close event to WAL at 1731836493577 (+85 ms)Running coprocessor post-close hooks at 1731836493580 (+3 ms)Closed at 1731836493580 2024-11-17T09:41:33,582 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=78abf17b9dde42332f107c0d3469140f, regionState=CLOSED 2024-11-17T09:41:33,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:41:33,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T09:41:33,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 78abf17b9dde42332f107c0d3469140f, server=70e37c475432,40671,1731836432285 in 1.9040 sec 2024-11-17T09:41:33,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-17T09:41:33,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78abf17b9dde42332f107c0d3469140f, UNASSIGN in 1.9080 sec 2024-11-17T09:41:33,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T09:41:33,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9110 sec 2024-11-17T09:41:33,590 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836493590"}]},"ts":"1731836493590"} 2024-11-17T09:41:33,591 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T09:41:33,591 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T09:41:33,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9200 sec 2024-11-17T09:41:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T09:41:33,820 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:33,821 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T09:41:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:33,822 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T09:41:33,823 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:33,825 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,828 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/recovered.edits] 2024-11-17T09:41:33,830 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/47004679debe41cab69251e161c02161 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/47004679debe41cab69251e161c02161 2024-11-17T09:41:33,831 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/51b72ec41e944281bf483193cac0d30e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/51b72ec41e944281bf483193cac0d30e 2024-11-17T09:41:33,832 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/76e0fb00f9634afeae49f7c0f9b966fe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/76e0fb00f9634afeae49f7c0f9b966fe 2024-11-17T09:41:33,833 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/8ede896323ff4ee28963f7df7a2de4e3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/A/8ede896323ff4ee28963f7df7a2de4e3 2024-11-17T09:41:33,835 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0912dad382f847689717ba657573a8f9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/0912dad382f847689717ba657573a8f9 2024-11-17T09:41:33,836 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/12ac92868f2c4332acf2aa0ea1058798 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/12ac92868f2c4332acf2aa0ea1058798 2024-11-17T09:41:33,837 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/50b3a962628e447bbfbac07407a9fca8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/50b3a962628e447bbfbac07407a9fca8 2024-11-17T09:41:33,837 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/58cbfe837ec0455aad5613151223bdec to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/B/58cbfe837ec0455aad5613151223bdec 2024-11-17T09:41:33,839 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/12b8986f4a4842529a683f1a394ae614 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/12b8986f4a4842529a683f1a394ae614 2024-11-17T09:41:33,841 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7e6e3c77829c47168c923217880ae0ef to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/7e6e3c77829c47168c923217880ae0ef 2024-11-17T09:41:33,842 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8c015c3c2b1e4445b44bda0f04c0a04a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/8c015c3c2b1e4445b44bda0f04c0a04a 2024-11-17T09:41:33,843 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c8dd418684874f4b983b1723656fef95 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/C/c8dd418684874f4b983b1723656fef95 2024-11-17T09:41:33,845 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/recovered.edits/628.seqid to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f/recovered.edits/628.seqid 2024-11-17T09:41:33,845 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,846 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T09:41:33,846 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T09:41:33,847 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T09:41:33,850 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117065e2e079c46475ba0813edcad631029_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117065e2e079c46475ba0813edcad631029_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,851 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111715d280a12278400a8616cbee4500dba1_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111715d280a12278400a8616cbee4500dba1_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,852 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171e2ae2a30589489a9ea3c4d9459f02ac_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171e2ae2a30589489a9ea3c4d9459f02ac_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,853 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172a61c3c67a654d498e27524cf887b0ed_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172a61c3c67a654d498e27524cf887b0ed_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,854 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172b9fee0a591242f69966e14cf646fe03_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172b9fee0a591242f69966e14cf646fe03_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,855 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e61dda1b0fb4b4383b319b81b3e1b3b_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172e61dda1b0fb4b4383b319b81b3e1b3b_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,856 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111734f2ff5b28744bd880e808530b202ade_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111734f2ff5b28744bd880e808530b202ade_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,857 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173b1c8ba512f54505b58ac122316e3e02_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173b1c8ba512f54505b58ac122316e3e02_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,858 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173c054931c88443059f6578831fc5068f_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173c054931c88443059f6578831fc5068f_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,859 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117404123b99b6c402d9f7d8e16689d7b19_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117404123b99b6c402d9f7d8e16689d7b19_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,860 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177535c91b2c3f4b009a53b2789d274494_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177535c91b2c3f4b009a53b2789d274494_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,861 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111776eae4e583f84d54a0b7b28f77a57ef7_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111776eae4e583f84d54a0b7b28f77a57ef7_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,862 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177e70411a108c42f0bfefe0732d6798fc_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177e70411a108c42f0bfefe0732d6798fc_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,863 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178ea88b4da4e940eeae339140a798699b_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178ea88b4da4e940eeae339140a798699b_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,864 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a5a84de21d6c4e048094b9d10b40c7ee_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a5a84de21d6c4e048094b9d10b40c7ee_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,865 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d05d9144908b42dc96bfbaa94d536227_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d05d9144908b42dc96bfbaa94d536227_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,866 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d368ee5276244a9b91eadf1914a24026_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d368ee5276244a9b91eadf1914a24026_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,867 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e3c834b4236f4f999aa4dabfadabcc5b_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e3c834b4236f4f999aa4dabfadabcc5b_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,868 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ea00dab177d1458b83fe5366f9f9bc26_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ea00dab177d1458b83fe5366f9f9bc26_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,869 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ebee650d799b4622b304eedad1cd28c3_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ebee650d799b4622b304eedad1cd28c3_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,870 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f213aadc65b24e8d942f5233dae9f4f1_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f213aadc65b24e8d942f5233dae9f4f1_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f2b8b6dc8e2049de97497e9056b839b7_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f2b8b6dc8e2049de97497e9056b839b7_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f5fa5f8889ef447aa2096318202711dc_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f5fa5f8889ef447aa2096318202711dc_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f9e6ef496d004d45990646f7b1efbe58_78abf17b9dde42332f107c0d3469140f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f9e6ef496d004d45990646f7b1efbe58_78abf17b9dde42332f107c0d3469140f 2024-11-17T09:41:33,873 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T09:41:33,875 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:33,877 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T09:41:33,879 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T09:41:33,880 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:33,880 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T09:41:33,880 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731836493880"}]},"ts":"9223372036854775807"} 2024-11-17T09:41:33,882 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T09:41:33,882 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 78abf17b9dde42332f107c0d3469140f, NAME => 'TestAcidGuarantees,,1731836462652.78abf17b9dde42332f107c0d3469140f.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T09:41:33,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T09:41:33,882 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731836493882"}]},"ts":"9223372036854775807"} 2024-11-17T09:41:33,883 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T09:41:33,884 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:33,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-17T09:41:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T09:41:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T09:41:33,930 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:33,940 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246 (was 246), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 441) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3319 (was 3124) - AvailableMemoryMB LEAK? - 2024-11-17T09:41:33,948 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=3318 2024-11-17T09:41:33,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:41:33,949 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:41:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T09:41:33,951 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:41:33,951 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:33,951 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-17T09:41:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T09:41:33,952 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:41:33,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742097_1273 (size=1061) 2024-11-17T09:41:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T09:41:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T09:41:34,358 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:41:34,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742098_1274 (size=53) 2024-11-17T09:41:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T09:41:34,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:41:34,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7ef56adb9d9541a35f7ad38e99d99e41, disabling compactions & flushes 2024-11-17T09:41:34,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:34,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:34,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. after waiting 0 ms 2024-11-17T09:41:34,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:34,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:34,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7ef56adb9d9541a35f7ad38e99d99e41: Waiting for close lock at 1731836494764Disabling compacts and flushes for region at 1731836494764Disabling writes for close at 1731836494764Writing region close event to WAL at 1731836494764Closed at 1731836494764 2024-11-17T09:41:34,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:41:34,766 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731836494765"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836494765"}]},"ts":"1731836494765"} 2024-11-17T09:41:34,767 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:41:34,768 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:41:34,768 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836494768"}]},"ts":"1731836494768"} 2024-11-17T09:41:34,769 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T09:41:34,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, ASSIGN}] 2024-11-17T09:41:34,770 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, ASSIGN 2024-11-17T09:41:34,771 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, ASSIGN; state=OFFLINE, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=false 2024-11-17T09:41:34,921 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7ef56adb9d9541a35f7ad38e99d99e41, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:34,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, ASSIGN because future has completed 2024-11-17T09:41:34,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:41:35,077 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:35,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:41:35,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:41:35,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,079 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,080 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:35,080 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ef56adb9d9541a35f7ad38e99d99e41 columnFamilyName A 2024-11-17T09:41:35,080 DEBUG [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:35,080 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.HStore(327): Store=7ef56adb9d9541a35f7ad38e99d99e41/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:35,080 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,081 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:35,081 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ef56adb9d9541a35f7ad38e99d99e41 columnFamilyName B 2024-11-17T09:41:35,082 DEBUG [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:35,082 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.HStore(327): Store=7ef56adb9d9541a35f7ad38e99d99e41/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:35,082 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,083 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:41:35,083 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ef56adb9d9541a35f7ad38e99d99e41 columnFamilyName C 2024-11-17T09:41:35,083 DEBUG [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:41:35,083 INFO [StoreOpener-7ef56adb9d9541a35f7ad38e99d99e41-1 {}] regionserver.HStore(327): Store=7ef56adb9d9541a35f7ad38e99d99e41/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:41:35,083 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:35,083 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,084 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,084 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,085 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,085 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,085 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:41:35,086 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,088 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:41:35,088 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 7ef56adb9d9541a35f7ad38e99d99e41; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75155505, jitterRate=0.1199042946100235}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:41:35,088 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:35,089 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 7ef56adb9d9541a35f7ad38e99d99e41: Running coprocessor pre-open hook at 1731836495077Writing region info on filesystem at 1731836495077Initializing all the Stores at 1731836495078 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836495078Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836495078Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836495078Cleaning up temporary data from old regions at 1731836495085 (+7 ms)Running coprocessor post-open hooks at 1731836495088 (+3 ms)Region opened successfully at 1731836495089 (+1 ms) 2024-11-17T09:41:35,089 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., pid=61, masterSystemTime=1731836495074 2024-11-17T09:41:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T09:41:35,091 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:35,091 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:35,091 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7ef56adb9d9541a35f7ad38e99d99e41, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:41:35,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:41:35,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T09:41:35,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 in 170 msec 2024-11-17T09:41:35,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-17T09:41:35,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, ASSIGN in 325 msec 2024-11-17T09:41:35,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:41:35,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836495097"}]},"ts":"1731836495097"} 2024-11-17T09:41:35,098 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T09:41:35,099 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:41:35,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-17T09:41:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-17T09:41:36,100 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:36,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e91c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,102 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732df114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,104 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad70db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,105 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,107 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:41:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,44533,1731836431485 2024-11-17T09:41:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3080f348 2024-11-17T09:41:36,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:41:36,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:41:36,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,111 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3beceed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,112 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65702b7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,114 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7719a451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,116 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33877523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,117 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,118 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c97d3b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,119 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fc096e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,121 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48575900, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,122 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272f7658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,124 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2a071a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,126 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a40942d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,128 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e125a37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f282e99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,131 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a0df959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,133 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e311f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,135 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653e0eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,136 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f68f839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,138 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f96fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,140 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f881e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:41:36,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:41:36,142 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:41:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:41:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:41:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358f8f26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:41:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:41:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:36,143 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:41:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad80ff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:41:36,144 DEBUG [Thread-1254 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,148 DEBUG [Thread-1269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,149 DEBUG [Thread-1267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,150 DEBUG [Thread-1265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,153 DEBUG [Thread-1265 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,153 DEBUG [Thread-1265 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,154 DEBUG [Thread-1273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,157 DEBUG [Thread-1271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,160 DEBUG [Thread-1258 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,167 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-17T09:41:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T09:41:36,170 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:36,170 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:36,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:36,174 DEBUG [Thread-1260 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,174 DEBUG [Thread-1256 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,175 DEBUG [Thread-1256 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,175 DEBUG [Thread-1262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:41:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,176 DEBUG [Thread-1256 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:41:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:41:36,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:41:36,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:41:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:41:36,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6543ee266b1c46ceb853d8c61f6a8954 is 50, key is test_row_0/A:col10/1731836496155/Put/seqid=0 2024-11-17T09:41:36,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731836496199, max timestamp=1731836496215Num uniques 859; 2024-11-17T09:41:36,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:36,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731836496199, max timestamp=1731836496215Num uniques 859; 2024-11-17T09:41:36,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:36,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836496199, max timestamp=1731836496215Num uniques 859; 2024-11-17T09:41:36,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836556247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836556249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836556249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836556251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742099_1275 (size=14341) 2024-11-17T09:41:36,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836556256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T09:41:36,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836556360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836556361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836556361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836556361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836556370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:36,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T09:41:36,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836556570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836556571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836556571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836556571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836556580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:36,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6543ee266b1c46ceb853d8c61f6a8954 2024-11-17T09:41:36,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/4bf11d736dd24bb09dfd6737b6d4892e is 50, key is test_row_0/B:col10/1731836496155/Put/seqid=0 2024-11-17T09:41:36,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742100_1276 (size=12001) 2024-11-17T09:41:36,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/4bf11d736dd24bb09dfd6737b6d4892e 2024-11-17T09:41:36,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/346d29dc64c94a20b61f173e51613f9b is 50, key is test_row_0/C:col10/1731836496155/Put/seqid=0 2024-11-17T09:41:36,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742101_1277 (size=12001) 2024-11-17T09:41:36,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/346d29dc64c94a20b61f173e51613f9b 2024-11-17T09:41:36,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6543ee266b1c46ceb853d8c61f6a8954 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6543ee266b1c46ceb853d8c61f6a8954 2024-11-17T09:41:36,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6543ee266b1c46ceb853d8c61f6a8954, entries=200, sequenceid=11, filesize=14.0 K 2024-11-17T09:41:36,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/4bf11d736dd24bb09dfd6737b6d4892e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bf11d736dd24bb09dfd6737b6d4892e 2024-11-17T09:41:36,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bf11d736dd24bb09dfd6737b6d4892e, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T09:41:36,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/346d29dc64c94a20b61f173e51613f9b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/346d29dc64c94a20b61f173e51613f9b 2024-11-17T09:41:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T09:41:36,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/346d29dc64c94a20b61f173e51613f9b, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T09:41:36,815 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-17T09:41:36,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 7ef56adb9d9541a35f7ad38e99d99e41 in 634ms, sequenceid=11, compaction requested=false 2024-11-17T09:41:36,816 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T09:41:36,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:36,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836496237, max timestamp=1731836496885Num uniques 1068; 2024-11-17T09:41:36,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731836496237, max timestamp=1731836496885Num uniques 1068; 2024-11-17T09:41:36,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T09:41:36,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-17T09:41:36,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731836496237, max timestamp=1731836496885Num uniques 1018; 2024-11-17T09:41:36,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-17T09:41:36,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=145.36 KB heapSize=415.77 KB 2024-11-17T09:41:36,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:36,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:36,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:36,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/9549c2046eb8408c98fe912da7b0c7cb is 50, key is test_row_0/A:col10/1731836496237/Put/seqid=0 2024-11-17T09:41:36,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:36,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:36,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742102_1278 (size=12001) 2024-11-17T09:41:36,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/9549c2046eb8408c98fe912da7b0c7cb 2024-11-17T09:41:36,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/38907c5d6e3843429437c52eb633ac52 is 50, key is test_row_0/B:col10/1731836496237/Put/seqid=0 2024-11-17T09:41:36,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836496931, max timestamp=1731836496965Num uniques 859; 2024-11-17T09:41:36,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:36,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836496931, max timestamp=1731836496965Num uniques 859; 2024-11-17T09:41:36,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:36,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836496931, max timestamp=1731836496965Num uniques 859; 2024-11-17T09:41:36,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:41:36,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742103_1279 (size=12001) 2024-11-17T09:41:36,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/38907c5d6e3843429437c52eb633ac52 2024-11-17T09:41:37,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/97686f99bd6a45d6b84a1315aac122c3 is 50, key is test_row_0/C:col10/1731836496237/Put/seqid=0 2024-11-17T09:41:37,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836497017, max timestamp=1731836497030Num uniques 1118; 2024-11-17T09:41:37,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T09:41:37,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836497017, max timestamp=1731836497030Num uniques 1118; 2024-11-17T09:41:37,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T09:41:37,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836497017, max timestamp=1731836497030Num uniques 1118; 2024-11-17T09:41:37,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T09:41:37,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742104_1280 (size=12001) 2024-11-17T09:41:37,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/97686f99bd6a45d6b84a1315aac122c3 2024-11-17T09:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/9549c2046eb8408c98fe912da7b0c7cb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9549c2046eb8408c98fe912da7b0c7cb 2024-11-17T09:41:37,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9549c2046eb8408c98fe912da7b0c7cb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T09:41:37,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/38907c5d6e3843429437c52eb633ac52 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38907c5d6e3843429437c52eb633ac52 2024-11-17T09:41:37,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38907c5d6e3843429437c52eb633ac52, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T09:41:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/97686f99bd6a45d6b84a1315aac122c3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/97686f99bd6a45d6b84a1315aac122c3 2024-11-17T09:41:37,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/97686f99bd6a45d6b84a1315aac122c3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T09:41:37,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.18 KB/35004, heapSize ~86.69 KB/88768, currentSize=120.76 KB/123660 for 7ef56adb9d9541a35f7ad38e99d99e41 in 184ms, sequenceid=50, compaction requested=false 2024-11-17T09:41:37,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:37,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=127.47 KB heapSize=364.22 KB 2024-11-17T09:41:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:37,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:37,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:37,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:37,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/bc316a652e4c4150a64197e9143f5d86 is 50, key is test_row_0/A:col10/1731836497048/Put/seqid=0 2024-11-17T09:41:37,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836557104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836557106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836557109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836557114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836557125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742105_1281 (size=12001) 2024-11-17T09:41:37,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/bc316a652e4c4150a64197e9143f5d86 2024-11-17T09:41:37,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/7a5e712f740c4efba87563a045f4a173 is 50, key is test_row_0/B:col10/1731836497048/Put/seqid=0 2024-11-17T09:41:37,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742106_1282 (size=12001) 2024-11-17T09:41:37,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/7a5e712f740c4efba87563a045f4a173 2024-11-17T09:41:37,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836557230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836557230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836557230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836557230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/56089bb144c341d6a4d3d6a73a030c73 is 50, key is test_row_0/C:col10/1731836497048/Put/seqid=0 2024-11-17T09:41:37,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:37,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836557240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:37,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742107_1283 (size=12001) 2024-11-17T09:41:37,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/56089bb144c341d6a4d3d6a73a030c73 2024-11-17T09:41:37,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/bc316a652e4c4150a64197e9143f5d86 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/bc316a652e4c4150a64197e9143f5d86 2024-11-17T09:41:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T09:41:37,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/bc316a652e4c4150a64197e9143f5d86, entries=150, sequenceid=103, filesize=11.7 K 2024-11-17T09:41:37,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/7a5e712f740c4efba87563a045f4a173 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7a5e712f740c4efba87563a045f4a173 2024-11-17T09:41:37,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7a5e712f740c4efba87563a045f4a173, entries=150, sequenceid=103, filesize=11.7 K 2024-11-17T09:41:37,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/56089bb144c341d6a4d3d6a73a030c73 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/56089bb144c341d6a4d3d6a73a030c73 2024-11-17T09:41:37,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/56089bb144c341d6a4d3d6a73a030c73, entries=150, sequenceid=103, filesize=11.7 K 2024-11-17T09:41:37,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~364.17 KB/372912, currentSize=60.38 KB/61830 for 7ef56adb9d9541a35f7ad38e99d99e41 in 258ms, sequenceid=103, compaction requested=true 2024-11-17T09:41:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:37,333 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:37,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:37,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:41:37,334 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:37,335 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:37,335 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/B is initiating minor compaction (all files) 2024-11-17T09:41:37,335 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/B in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,335 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bf11d736dd24bb09dfd6737b6d4892e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38907c5d6e3843429437c52eb633ac52, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7a5e712f740c4efba87563a045f4a173] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=35.2 K 2024-11-17T09:41:37,335 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:37,335 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/A is initiating minor compaction (all files) 2024-11-17T09:41:37,335 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/A in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,335 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6543ee266b1c46ceb853d8c61f6a8954, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9549c2046eb8408c98fe912da7b0c7cb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/bc316a652e4c4150a64197e9143f5d86] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=37.4 K 2024-11-17T09:41:37,336 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf11d736dd24bb09dfd6737b6d4892e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731836496155 2024-11-17T09:41:37,336 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 38907c5d6e3843429437c52eb633ac52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731836496237 2024-11-17T09:41:37,336 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6543ee266b1c46ceb853d8c61f6a8954, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731836496155 2024-11-17T09:41:37,337 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5e712f740c4efba87563a045f4a173, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731836497048 2024-11-17T09:41:37,337 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9549c2046eb8408c98fe912da7b0c7cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731836496237 2024-11-17T09:41:37,338 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc316a652e4c4150a64197e9143f5d86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731836497048 2024-11-17T09:41:37,355 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:37,355 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/5af87c0831e9420c8402ac48d6a761c4 is 50, key is test_row_0/B:col10/1731836497048/Put/seqid=0 2024-11-17T09:41:37,357 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#A#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:37,358 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b58624f8be144b35a314b9305c1c820b is 50, key is test_row_0/A:col10/1731836497048/Put/seqid=0 2024-11-17T09:41:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742108_1284 (size=12104) 2024-11-17T09:41:37,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-17T09:41:37,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,407 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T09:41:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:37,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742109_1285 (size=12104) 2024-11-17T09:41:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:37,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/a19cb6509e2c486ab667c5f564081b90 is 50, key is test_row_0/A:col10/1731836497080/Put/seqid=0 2024-11-17T09:41:37,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742110_1286 (size=12001) 2024-11-17T09:41:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:37,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:37,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836557469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836557470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836557470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836557477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836557477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836557580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836557580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836557581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836557590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836557590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,790 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b58624f8be144b35a314b9305c1c820b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b58624f8be144b35a314b9305c1c820b 2024-11-17T09:41:37,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836557790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836557790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836557790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,796 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/A of 7ef56adb9d9541a35f7ad38e99d99e41 into b58624f8be144b35a314b9305c1c820b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,796 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,796 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/A, priority=13, startTime=1731836497333; duration=0sec 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,796 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:37,796 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:A 2024-11-17T09:41:37,796 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:37,797 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:37,797 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/C is initiating minor compaction (all files) 2024-11-17T09:41:37,798 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/C in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,798 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/346d29dc64c94a20b61f173e51613f9b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/97686f99bd6a45d6b84a1315aac122c3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/56089bb144c341d6a4d3d6a73a030c73] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=35.2 K 2024-11-17T09:41:37,798 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 346d29dc64c94a20b61f173e51613f9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731836496155 2024-11-17T09:41:37,799 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97686f99bd6a45d6b84a1315aac122c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731836496237 2024-11-17T09:41:37,799 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56089bb144c341d6a4d3d6a73a030c73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731836497048 2024-11-17T09:41:37,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836557800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836557800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:37,815 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/5af87c0831e9420c8402ac48d6a761c4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/5af87c0831e9420c8402ac48d6a761c4 2024-11-17T09:41:37,821 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/B of 7ef56adb9d9541a35f7ad38e99d99e41 into 5af87c0831e9420c8402ac48d6a761c4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:37,821 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:37,821 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/B, priority=13, startTime=1731836497333; duration=0sec 2024-11-17T09:41:37,821 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:37,821 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:B 2024-11-17T09:41:37,827 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/a19cb6509e2c486ab667c5f564081b90 2024-11-17T09:41:37,835 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#C#compaction#237 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:37,836 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/e51f07f6596d4bfb854b7e7757f4e226 is 50, key is test_row_0/C:col10/1731836497048/Put/seqid=0 2024-11-17T09:41:37,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6d5c79fe6dd040089e61548f2c4c5535 is 50, key is test_row_0/B:col10/1731836497080/Put/seqid=0 2024-11-17T09:41:37,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742111_1287 (size=12104) 2024-11-17T09:41:37,910 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/e51f07f6596d4bfb854b7e7757f4e226 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/e51f07f6596d4bfb854b7e7757f4e226 2024-11-17T09:41:37,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742112_1288 (size=12001) 2024-11-17T09:41:37,912 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6d5c79fe6dd040089e61548f2c4c5535 2024-11-17T09:41:37,917 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/C of 7ef56adb9d9541a35f7ad38e99d99e41 into e51f07f6596d4bfb854b7e7757f4e226(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:37,917 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:37,917 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/C, priority=13, startTime=1731836497333; duration=0sec 2024-11-17T09:41:37,917 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:37,917 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:C 2024-11-17T09:41:37,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/29ff9ca3f32b4626a10f03a9bc13258c is 50, key is test_row_0/C:col10/1731836497080/Put/seqid=0 2024-11-17T09:41:37,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742113_1289 (size=12001) 2024-11-17T09:41:37,949 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/29ff9ca3f32b4626a10f03a9bc13258c 2024-11-17T09:41:37,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/a19cb6509e2c486ab667c5f564081b90 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/a19cb6509e2c486ab667c5f564081b90 2024-11-17T09:41:37,960 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/a19cb6509e2c486ab667c5f564081b90, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T09:41:37,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6d5c79fe6dd040089e61548f2c4c5535 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6d5c79fe6dd040089e61548f2c4c5535 2024-11-17T09:41:37,966 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6d5c79fe6dd040089e61548f2c4c5535, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T09:41:37,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/29ff9ca3f32b4626a10f03a9bc13258c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/29ff9ca3f32b4626a10f03a9bc13258c 2024-11-17T09:41:37,971 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/29ff9ca3f32b4626a10f03a9bc13258c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-17T09:41:37,972 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7ef56adb9d9541a35f7ad38e99d99e41 in 565ms, sequenceid=115, compaction requested=false 2024-11-17T09:41:37,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:37,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:37,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-17T09:41:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-17T09:41:37,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T09:41:37,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8040 sec 2024-11-17T09:41:37,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.8100 sec 2024-11-17T09:41:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:38,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836497467, max timestamp=1731836498101Num uniques 859; 2024-11-17T09:41:38,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:38,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836497467, max timestamp=1731836498101Num uniques 859; 2024-11-17T09:41:38,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:38,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-17T09:41:38,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:38,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836497467, max timestamp=1731836498101Num uniques 859; 2024-11-17T09:41:38,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:38,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:38,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:38,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:38,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:38,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:38,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b75afb465cc546069d59ca634b8a0c1b is 50, key is test_row_0/A:col10/1731836498105/Put/seqid=0 2024-11-17T09:41:38,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742114_1290 (size=12151) 2024-11-17T09:41:38,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b75afb465cc546069d59ca634b8a0c1b 2024-11-17T09:41:38,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836498143, max timestamp=1731836498146Num uniques 859; 2024-11-17T09:41:38,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:38,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836498143, max timestamp=1731836498146Num uniques 859; 2024-11-17T09:41:38,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:38,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836498143, max timestamp=1731836498147Num uniques 859; 2024-11-17T09:41:38,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:38,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/7ab64349dadc45f4b797223e449d9663 is 50, key is test_row_0/B:col10/1731836498105/Put/seqid=0 2024-11-17T09:41:38,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731836498162, max timestamp=1731836498195Num uniques 1018; 2024-11-17T09:41:38,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731836498162, max timestamp=1731836498195Num uniques 1018; 2024-11-17T09:41:38,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T09:41:38,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T09:41:38,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731836498162, max timestamp=1731836498195Num uniques 1018; 2024-11-17T09:41:38,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T09:41:38,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742115_1291 (size=12151) 2024-11-17T09:41:38,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/7ab64349dadc45f4b797223e449d9663 2024-11-17T09:41:38,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/26c7ab6d45214d699a88e3504fef3f84 is 50, key is test_row_0/C:col10/1731836498105/Put/seqid=0 2024-11-17T09:41:38,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742116_1292 (size=12151) 2024-11-17T09:41:38,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/26c7ab6d45214d699a88e3504fef3f84 2024-11-17T09:41:38,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731836498215, max timestamp=1731836498239Num uniques 1127; 2024-11-17T09:41:38,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-17T09:41:38,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b75afb465cc546069d59ca634b8a0c1b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b75afb465cc546069d59ca634b8a0c1b 2024-11-17T09:41:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836498227, max timestamp=1731836498239Num uniques 1127; 2024-11-17T09:41:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-17T09:41:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836498227, max timestamp=1731836498239Num uniques 1127; 2024-11-17T09:41:38,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-17T09:41:38,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b75afb465cc546069d59ca634b8a0c1b, entries=150, sequenceid=141, filesize=11.9 K 2024-11-17T09:41:38,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/7ab64349dadc45f4b797223e449d9663 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7ab64349dadc45f4b797223e449d9663 2024-11-17T09:41:38,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7ab64349dadc45f4b797223e449d9663, entries=150, sequenceid=141, filesize=11.9 K 2024-11-17T09:41:38,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/26c7ab6d45214d699a88e3504fef3f84 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/26c7ab6d45214d699a88e3504fef3f84 2024-11-17T09:41:38,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/26c7ab6d45214d699a88e3504fef3f84, entries=150, sequenceid=141, filesize=11.9 K 2024-11-17T09:41:38,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=87.22 KB/89310 for 7ef56adb9d9541a35f7ad38e99d99e41 in 168ms, sequenceid=141, compaction requested=true 2024-11-17T09:41:38,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:38,274 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:38,274 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:38,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=100.63 KB heapSize=279 KB 2024-11-17T09:41:38,275 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:38,276 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/A is initiating minor compaction (all files) 2024-11-17T09:41:38,276 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/A in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,276 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b58624f8be144b35a314b9305c1c820b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/a19cb6509e2c486ab667c5f564081b90, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b75afb465cc546069d59ca634b8a0c1b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=35.4 K 2024-11-17T09:41:38,276 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:38,276 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b58624f8be144b35a314b9305c1c820b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731836497048 2024-11-17T09:41:38,276 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/B is initiating minor compaction (all files) 2024-11-17T09:41:38,276 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/B in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,276 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/5af87c0831e9420c8402ac48d6a761c4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6d5c79fe6dd040089e61548f2c4c5535, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7ab64349dadc45f4b797223e449d9663] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=35.4 K 2024-11-17T09:41:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:38,277 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a19cb6509e2c486ab667c5f564081b90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731836497080 2024-11-17T09:41:38,277 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af87c0831e9420c8402ac48d6a761c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731836497048 2024-11-17T09:41:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:38,278 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b75afb465cc546069d59ca634b8a0c1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836497475 2024-11-17T09:41:38,278 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5c79fe6dd040089e61548f2c4c5535, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731836497080 2024-11-17T09:41:38,278 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab64349dadc45f4b797223e449d9663, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836497475 2024-11-17T09:41:38,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/fdf8d39e29fd419caf9b7db4d6aacb0c is 50, key is test_row_0/A:col10/1731836498271/Put/seqid=0 2024-11-17T09:41:38,304 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:38,305 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/9aa1869af1764387946f859d315ea0de is 50, key is test_row_0/B:col10/1731836498105/Put/seqid=0 2024-11-17T09:41:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836558307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T09:41:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,321 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:38,321 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-17T09:41:38,323 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T09:41:38,324 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:38,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:38,327 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#A#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:38,327 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/5b2129c121424c64a37faedcc7a1b8cb is 50, key is test_row_0/A:col10/1731836498105/Put/seqid=0 2024-11-17T09:41:38,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836558311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836558311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836558315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836558319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742118_1294 (size=14541) 2024-11-17T09:41:38,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742117_1293 (size=12357) 2024-11-17T09:41:38,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742119_1295 (size=12357) 2024-11-17T09:41:38,392 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/5b2129c121424c64a37faedcc7a1b8cb as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/5b2129c121424c64a37faedcc7a1b8cb 2024-11-17T09:41:38,392 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/9aa1869af1764387946f859d315ea0de as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/9aa1869af1764387946f859d315ea0de 2024-11-17T09:41:38,399 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/B of 7ef56adb9d9541a35f7ad38e99d99e41 into 9aa1869af1764387946f859d315ea0de(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:38,399 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/A of 7ef56adb9d9541a35f7ad38e99d99e41 into 5b2129c121424c64a37faedcc7a1b8cb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:38,399 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:38,399 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:38,399 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/B, priority=13, startTime=1731836498274; duration=0sec 2024-11-17T09:41:38,399 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/A, priority=13, startTime=1731836498274; duration=0sec 2024-11-17T09:41:38,399 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:38,399 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:A 2024-11-17T09:41:38,399 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:38,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:38,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:B 2024-11-17T09:41:38,402 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:38,403 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/C is initiating minor compaction (all files) 2024-11-17T09:41:38,403 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/C in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,403 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/e51f07f6596d4bfb854b7e7757f4e226, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/29ff9ca3f32b4626a10f03a9bc13258c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/26c7ab6d45214d699a88e3504fef3f84] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=35.4 K 2024-11-17T09:41:38,404 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e51f07f6596d4bfb854b7e7757f4e226, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731836497048 2024-11-17T09:41:38,404 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ff9ca3f32b4626a10f03a9bc13258c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731836497080 2024-11-17T09:41:38,404 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 26c7ab6d45214d699a88e3504fef3f84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836497475 2024-11-17T09:41:38,414 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:38,414 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/c98bd0b28bbf4a33ba739aa04a25af52 is 50, key is test_row_0/C:col10/1731836498105/Put/seqid=0 2024-11-17T09:41:38,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742120_1296 (size=12357) 2024-11-17T09:41:38,425 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/c98bd0b28bbf4a33ba739aa04a25af52 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c98bd0b28bbf4a33ba739aa04a25af52 2024-11-17T09:41:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T09:41:38,431 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/C of 7ef56adb9d9541a35f7ad38e99d99e41 into c98bd0b28bbf4a33ba739aa04a25af52(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:38,431 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:38,431 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/C, priority=13, startTime=1731836498274; duration=0sec 2024-11-17T09:41:38,431 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:38,431 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:C 2024-11-17T09:41:38,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836558429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836558440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836558440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836558440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836558440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T09:41:38,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:38,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T09:41:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T09:41:38,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836558640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836558649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836558649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836558650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836558650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/fdf8d39e29fd419caf9b7db4d6aacb0c 2024-11-17T09:41:38,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T09:41:38,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:38,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/afd5fc89dcc6494e8651ab333d3ae44f is 50, key is test_row_0/B:col10/1731836498271/Put/seqid=0 2024-11-17T09:41:38,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742121_1297 (size=12151) 2024-11-17T09:41:38,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T09:41:38,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:38,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:38,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T09:41:38,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836558960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836558960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836558960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836558960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836558960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T09:41:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:39,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:39,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:39,107 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:41:39,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/afd5fc89dcc6494e8651ab333d3ae44f 2024-11-17T09:41:39,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/bcba0fb22c8b40c4b0a762ac29449fe2 is 50, key is test_row_0/C:col10/1731836498271/Put/seqid=0 2024-11-17T09:41:39,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742122_1298 (size=12151) 2024-11-17T09:41:39,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/bcba0fb22c8b40c4b0a762ac29449fe2 2024-11-17T09:41:39,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/fdf8d39e29fd419caf9b7db4d6aacb0c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/fdf8d39e29fd419caf9b7db4d6aacb0c 2024-11-17T09:41:39,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T09:41:39,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:39,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:39,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:39,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/fdf8d39e29fd419caf9b7db4d6aacb0c, entries=200, sequenceid=206, filesize=14.2 K 2024-11-17T09:41:39,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/afd5fc89dcc6494e8651ab333d3ae44f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/afd5fc89dcc6494e8651ab333d3ae44f 2024-11-17T09:41:39,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/afd5fc89dcc6494e8651ab333d3ae44f, entries=150, sequenceid=206, filesize=11.9 K 2024-11-17T09:41:39,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/bcba0fb22c8b40c4b0a762ac29449fe2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bcba0fb22c8b40c4b0a762ac29449fe2 2024-11-17T09:41:39,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bcba0fb22c8b40c4b0a762ac29449fe2, entries=150, sequenceid=206, filesize=11.9 K 2024-11-17T09:41:39,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~278.95 KB/285648, currentSize=80.51 KB/82440 for 7ef56adb9d9541a35f7ad38e99d99e41 in 990ms, sequenceid=206, compaction requested=false 2024-11-17T09:41:39,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:39,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T09:41:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:39,402 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T09:41:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:39,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/144084db86c042dda7ba6aaad8cdf5c4 is 50, key is test_row_0/A:col10/1731836498316/Put/seqid=0 2024-11-17T09:41:39,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742123_1299 (size=12151) 2024-11-17T09:41:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T09:41:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:39,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:39,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836559488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836559494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836559493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836559495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836559497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836559599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836559611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836559611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836559611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836559611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836559810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836559820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836559820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836559829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,833 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/144084db86c042dda7ba6aaad8cdf5c4 2024-11-17T09:41:39,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836559829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:39,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/4bed662a49c346d58b72d6470e4e6b4f is 50, key is test_row_0/B:col10/1731836498316/Put/seqid=0 2024-11-17T09:41:39,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742124_1300 (size=12151) 2024-11-17T09:41:40,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836560120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836560129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836560129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836560140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836560140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,252 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/4bed662a49c346d58b72d6470e4e6b4f 2024-11-17T09:41:40,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/f30cc734a8fc4e2aaab124f12bd434cf is 50, key is test_row_0/C:col10/1731836498316/Put/seqid=0 2024-11-17T09:41:40,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742125_1301 (size=12151) 2024-11-17T09:41:40,291 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/f30cc734a8fc4e2aaab124f12bd434cf 2024-11-17T09:41:40,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/144084db86c042dda7ba6aaad8cdf5c4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/144084db86c042dda7ba6aaad8cdf5c4 2024-11-17T09:41:40,306 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/144084db86c042dda7ba6aaad8cdf5c4, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T09:41:40,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/4bed662a49c346d58b72d6470e4e6b4f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bed662a49c346d58b72d6470e4e6b4f 2024-11-17T09:41:40,314 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bed662a49c346d58b72d6470e4e6b4f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T09:41:40,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/f30cc734a8fc4e2aaab124f12bd434cf as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/f30cc734a8fc4e2aaab124f12bd434cf 2024-11-17T09:41:40,322 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/f30cc734a8fc4e2aaab124f12bd434cf, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T09:41:40,323 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 7ef56adb9d9541a35f7ad38e99d99e41 in 921ms, sequenceid=224, compaction requested=true 2024-11-17T09:41:40,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:40,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:40,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-17T09:41:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-17T09:41:40,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-17T09:41:40,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0010 sec 2024-11-17T09:41:40,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.0070 sec 2024-11-17T09:41:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T09:41:40,470 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:40,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-17T09:41:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T09:41:40,473 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:40,474 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:40,474 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T09:41:40,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T09:41:40,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:40,627 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T09:41:40,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:40,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:40,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:40,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:40,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:40,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:40,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:40,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/180b93172a434d6ea221f67cbd5b6bbe is 50, key is test_row_0/A:col10/1731836499484/Put/seqid=0 2024-11-17T09:41:40,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742126_1302 (size=12151) 2024-11-17T09:41:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836560656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836560655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836560664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836560665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836560665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836560770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836560770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836560780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836560780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836560780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T09:41:40,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836560980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836560980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836560990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836560991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836560991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,039 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/180b93172a434d6ea221f67cbd5b6bbe 2024-11-17T09:41:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/d7c1820b10c84a09a7cdaf80d454c755 is 50, key is test_row_0/B:col10/1731836499484/Put/seqid=0 2024-11-17T09:41:41,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742127_1303 (size=12151) 2024-11-17T09:41:41,095 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/d7c1820b10c84a09a7cdaf80d454c755 2024-11-17T09:41:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T09:41:41,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/9931f50bb8844b86bb5ef0fd69dc34fe is 50, key is test_row_0/C:col10/1731836499484/Put/seqid=0 2024-11-17T09:41:41,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742128_1304 (size=12151) 2024-11-17T09:41:41,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836561289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836561300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836561300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836561301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836561301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,531 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/9931f50bb8844b86bb5ef0fd69dc34fe 2024-11-17T09:41:41,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/180b93172a434d6ea221f67cbd5b6bbe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/180b93172a434d6ea221f67cbd5b6bbe 2024-11-17T09:41:41,551 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/180b93172a434d6ea221f67cbd5b6bbe, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T09:41:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/d7c1820b10c84a09a7cdaf80d454c755 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/d7c1820b10c84a09a7cdaf80d454c755 2024-11-17T09:41:41,561 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/d7c1820b10c84a09a7cdaf80d454c755, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T09:41:41,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/9931f50bb8844b86bb5ef0fd69dc34fe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/9931f50bb8844b86bb5ef0fd69dc34fe 2024-11-17T09:41:41,567 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/9931f50bb8844b86bb5ef0fd69dc34fe, entries=150, sequenceid=242, filesize=11.9 K 2024-11-17T09:41:41,568 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 7ef56adb9d9541a35f7ad38e99d99e41 in 941ms, sequenceid=242, compaction requested=true 2024-11-17T09:41:41,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:41,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:41,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-17T09:41:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-17T09:41:41,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1070 sec 2024-11-17T09:41:41,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T09:41:41,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-17T09:41:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T09:41:41,610 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:41,611 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-17T09:41:41,613 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T09:41:41,614 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:41,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T09:41:41,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T09:41:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:41,770 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:41:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:41,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:41,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/56588685c1bb42b9931b80c99e8bf017 is 50, key is test_row_0/A:col10/1731836500659/Put/seqid=0 2024-11-17T09:41:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:41,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:41,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742129_1305 (size=12201) 2024-11-17T09:41:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836561826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836561827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836561828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836561829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836561830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T09:41:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836561940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:41,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836562150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836562150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836562150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836562178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836562172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,205 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/56588685c1bb42b9931b80c99e8bf017 2024-11-17T09:41:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/8b4731f5eb41454f9477c1d90fe74690 is 50, key is test_row_0/B:col10/1731836500659/Put/seqid=0 2024-11-17T09:41:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T09:41:42,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742130_1306 (size=12201) 2024-11-17T09:41:42,259 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/8b4731f5eb41454f9477c1d90fe74690 2024-11-17T09:41:42,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/08d4034d160c4e3191c5224873b4791e is 50, key is test_row_0/C:col10/1731836500659/Put/seqid=0 2024-11-17T09:41:42,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742131_1307 (size=12201) 2024-11-17T09:41:42,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836562459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836562459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836562460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836562490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836562490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:42,678 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/08d4034d160c4e3191c5224873b4791e 2024-11-17T09:41:42,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/56588685c1bb42b9931b80c99e8bf017 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/56588685c1bb42b9931b80c99e8bf017 2024-11-17T09:41:42,692 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/56588685c1bb42b9931b80c99e8bf017, entries=150, sequenceid=258, filesize=11.9 K 2024-11-17T09:41:42,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/8b4731f5eb41454f9477c1d90fe74690 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/8b4731f5eb41454f9477c1d90fe74690 2024-11-17T09:41:42,698 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/8b4731f5eb41454f9477c1d90fe74690, entries=150, sequenceid=258, filesize=11.9 K 2024-11-17T09:41:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/08d4034d160c4e3191c5224873b4791e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/08d4034d160c4e3191c5224873b4791e 2024-11-17T09:41:42,702 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/08d4034d160c4e3191c5224873b4791e, entries=150, sequenceid=258, filesize=11.9 K 2024-11-17T09:41:42,703 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7ef56adb9d9541a35f7ad38e99d99e41 in 934ms, sequenceid=258, compaction requested=true 2024-11-17T09:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-17T09:41:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-17T09:41:42,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T09:41:42,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-11-17T09:41:42,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0960 sec 2024-11-17T09:41:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T09:41:42,750 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:42,750 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T09:41:42,752 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T09:41:42,753 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:42,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T09:41:42,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T09:41:42,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:42,906 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:41:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:42,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/dd0951db82804b45b7d4c7914ede5879 is 50, key is test_row_0/A:col10/1731836501825/Put/seqid=0 2024-11-17T09:41:42,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742132_1308 (size=9857) 2024-11-17T09:41:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:42,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:43,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836562996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836562997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836563000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836563001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836563010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T09:41:43,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836563110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836563110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836563110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,317 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/dd0951db82804b45b7d4c7914ede5879 2024-11-17T09:41:43,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836563320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836563321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836563321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/47e07514f57d4d6aafa2a3d2b8833c38 is 50, key is test_row_0/B:col10/1731836501825/Put/seqid=0 2024-11-17T09:41:43,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742133_1309 (size=9857) 2024-11-17T09:41:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T09:41:43,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836563630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836563630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836563630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:43,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:43,730 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/47e07514f57d4d6aafa2a3d2b8833c38 2024-11-17T09:41:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/8c2b5f5884e34b378daa91365cc41b7a is 50, key is test_row_0/C:col10/1731836501825/Put/seqid=0 2024-11-17T09:41:43,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742134_1310 (size=9857) 2024-11-17T09:41:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T09:41:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836564009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:44,012 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:44,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836564030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:44,032 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836564140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836564140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836564140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:44,162 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/8c2b5f5884e34b378daa91365cc41b7a 2024-11-17T09:41:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/dd0951db82804b45b7d4c7914ede5879 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/dd0951db82804b45b7d4c7914ede5879 2024-11-17T09:41:44,174 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/dd0951db82804b45b7d4c7914ede5879, entries=100, sequenceid=274, filesize=9.6 K 2024-11-17T09:41:44,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/47e07514f57d4d6aafa2a3d2b8833c38 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/47e07514f57d4d6aafa2a3d2b8833c38 2024-11-17T09:41:44,179 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/47e07514f57d4d6aafa2a3d2b8833c38, entries=100, sequenceid=274, filesize=9.6 K 2024-11-17T09:41:44,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/8c2b5f5884e34b378daa91365cc41b7a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8c2b5f5884e34b378daa91365cc41b7a 2024-11-17T09:41:44,186 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8c2b5f5884e34b378daa91365cc41b7a, entries=100, sequenceid=274, filesize=9.6 K 2024-11-17T09:41:44,186 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1280ms, sequenceid=274, compaction requested=true 2024-11-17T09:41:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:44,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T09:41:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T09:41:44,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T09:41:44,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-17T09:41:44,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4410 sec 2024-11-17T09:41:44,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T09:41:44,900 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:44,901 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:44,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T09:41:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T09:41:44,903 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:44,904 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:44,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:45,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T09:41:45,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T09:41:45,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:45,058 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:41:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:45,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/7a74c38ca2d042c5a98b1ad50dd69f46 is 50, key is test_row_0/A:col10/1731836502999/Put/seqid=0 2024-11-17T09:41:45,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742135_1311 (size=12301) 2024-11-17T09:41:45,082 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/7a74c38ca2d042c5a98b1ad50dd69f46 2024-11-17T09:41:45,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/3b0ddb10d2ec409cb7bec5f0af80b93c is 50, key is test_row_0/B:col10/1731836502999/Put/seqid=0 2024-11-17T09:41:45,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742136_1312 (size=12301) 2024-11-17T09:41:45,139 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/3b0ddb10d2ec409cb7bec5f0af80b93c 2024-11-17T09:41:45,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/83e62f46183849f483dc61dd37a583b3 is 50, key is test_row_0/C:col10/1731836502999/Put/seqid=0 2024-11-17T09:41:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:45,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:45,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742137_1313 (size=12301) 2024-11-17T09:41:45,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836565196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836565206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836565207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T09:41:45,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836565309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836565320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836565320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T09:41:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836565530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836565540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836565540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,582 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/83e62f46183849f483dc61dd37a583b3 2024-11-17T09:41:45,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/7a74c38ca2d042c5a98b1ad50dd69f46 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/7a74c38ca2d042c5a98b1ad50dd69f46 2024-11-17T09:41:45,591 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/7a74c38ca2d042c5a98b1ad50dd69f46, entries=150, sequenceid=290, filesize=12.0 K 2024-11-17T09:41:45,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/3b0ddb10d2ec409cb7bec5f0af80b93c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3b0ddb10d2ec409cb7bec5f0af80b93c 2024-11-17T09:41:45,597 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3b0ddb10d2ec409cb7bec5f0af80b93c, entries=150, sequenceid=290, filesize=12.0 K 2024-11-17T09:41:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/83e62f46183849f483dc61dd37a583b3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/83e62f46183849f483dc61dd37a583b3 2024-11-17T09:41:45,601 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/83e62f46183849f483dc61dd37a583b3, entries=150, sequenceid=290, filesize=12.0 K 2024-11-17T09:41:45,602 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7ef56adb9d9541a35f7ad38e99d99e41 in 544ms, sequenceid=290, compaction requested=true 2024-11-17T09:41:45,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:45,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:45,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T09:41:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T09:41:45,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T09:41:45,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 700 msec 2024-11-17T09:41:45,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 706 msec 2024-11-17T09:41:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:45,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:41:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:45,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/8bbddc08523a4cd1bae9df2ce1c6b550 is 50, key is test_row_0/A:col10/1731836505174/Put/seqid=0 2024-11-17T09:41:45,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742138_1314 (size=14741) 2024-11-17T09:41:45,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836565870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836565876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836565877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836565980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836565990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836565990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836566029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,032 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T09:41:46,040 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:46,040 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T09:41:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T09:41:46,042 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:46,042 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:46,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:46,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836566049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,055 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T09:41:46,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836566190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T09:41:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836566200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836566200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/8bbddc08523a4cd1bae9df2ce1c6b550 2024-11-17T09:41:46,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/18a55df54442450cb65836ccb2fde13b is 50, key is test_row_0/B:col10/1731836505174/Put/seqid=0 2024-11-17T09:41:46,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742139_1315 (size=12301) 2024-11-17T09:41:46,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T09:41:46,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:46,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T09:41:46,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T09:41:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836566500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836566510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836566510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:46,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T09:41:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/18a55df54442450cb65836ccb2fde13b 2024-11-17T09:41:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T09:41:46,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/2ea3875c3e0c4920b78a279319bf413b is 50, key is test_row_0/C:col10/1731836505174/Put/seqid=0 2024-11-17T09:41:46,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742140_1316 (size=12301) 2024-11-17T09:41:46,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T09:41:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T09:41:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:46,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:47,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836567010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:47,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836567020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:47,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836567020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:47,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:47,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/2ea3875c3e0c4920b78a279319bf413b 2024-11-17T09:41:47,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/8bbddc08523a4cd1bae9df2ce1c6b550 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/8bbddc08523a4cd1bae9df2ce1c6b550 2024-11-17T09:41:47,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/8bbddc08523a4cd1bae9df2ce1c6b550, entries=200, sequenceid=307, filesize=14.4 K 2024-11-17T09:41:47,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/18a55df54442450cb65836ccb2fde13b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/18a55df54442450cb65836ccb2fde13b 2024-11-17T09:41:47,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/18a55df54442450cb65836ccb2fde13b, entries=150, sequenceid=307, filesize=12.0 K 2024-11-17T09:41:47,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/2ea3875c3e0c4920b78a279319bf413b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2ea3875c3e0c4920b78a279319bf413b 2024-11-17T09:41:47,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2ea3875c3e0c4920b78a279319bf413b, entries=150, sequenceid=307, filesize=12.0 K 2024-11-17T09:41:47,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1253ms, sequenceid=307, compaction requested=true 2024-11-17T09:41:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:47,097 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-17T09:41:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:47,097 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-17T09:41:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:47,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:47,099 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 100300 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-17T09:41:47,099 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 95470 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-17T09:41:47,099 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/B is initiating minor compaction (all files) 2024-11-17T09:41:47,099 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/A is initiating minor compaction (all files) 2024-11-17T09:41:47,100 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/B in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:47,100 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/A in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:47,100 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/9aa1869af1764387946f859d315ea0de, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/afd5fc89dcc6494e8651ab333d3ae44f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bed662a49c346d58b72d6470e4e6b4f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/d7c1820b10c84a09a7cdaf80d454c755, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/8b4731f5eb41454f9477c1d90fe74690, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/47e07514f57d4d6aafa2a3d2b8833c38, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3b0ddb10d2ec409cb7bec5f0af80b93c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/18a55df54442450cb65836ccb2fde13b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=93.2 K 2024-11-17T09:41:47,100 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/5b2129c121424c64a37faedcc7a1b8cb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/fdf8d39e29fd419caf9b7db4d6aacb0c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/144084db86c042dda7ba6aaad8cdf5c4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/180b93172a434d6ea221f67cbd5b6bbe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/56588685c1bb42b9931b80c99e8bf017, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/dd0951db82804b45b7d4c7914ede5879, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/7a74c38ca2d042c5a98b1ad50dd69f46, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/8bbddc08523a4cd1bae9df2ce1c6b550] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=97.9 K 2024-11-17T09:41:47,101 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa1869af1764387946f859d315ea0de, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836497475 2024-11-17T09:41:47,102 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b2129c121424c64a37faedcc7a1b8cb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836497475 2024-11-17T09:41:47,102 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting afd5fc89dcc6494e8651ab333d3ae44f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731836498268 2024-11-17T09:41:47,102 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdf8d39e29fd419caf9b7db4d6aacb0c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731836498268 2024-11-17T09:41:47,103 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 144084db86c042dda7ba6aaad8cdf5c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731836498303 2024-11-17T09:41:47,103 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bed662a49c346d58b72d6470e4e6b4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731836498303 2024-11-17T09:41:47,103 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 180b93172a434d6ea221f67cbd5b6bbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731836499484 2024-11-17T09:41:47,103 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c1820b10c84a09a7cdaf80d454c755, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731836499484 2024-11-17T09:41:47,103 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56588685c1bb42b9931b80c99e8bf017, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731836500659 2024-11-17T09:41:47,103 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b4731f5eb41454f9477c1d90fe74690, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731836500659 2024-11-17T09:41:47,104 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 47e07514f57d4d6aafa2a3d2b8833c38, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731836501825 2024-11-17T09:41:47,104 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0951db82804b45b7d4c7914ede5879, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731836501825 2024-11-17T09:41:47,104 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b0ddb10d2ec409cb7bec5f0af80b93c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731836502987 2024-11-17T09:41:47,104 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a74c38ca2d042c5a98b1ad50dd69f46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731836502987 2024-11-17T09:41:47,104 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a55df54442450cb65836ccb2fde13b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731836505174 2024-11-17T09:41:47,104 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbddc08523a4cd1bae9df2ce1c6b550, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731836505174 2024-11-17T09:41:47,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T09:41:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:47,114 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T09:41:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:47,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:47,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/4cbb5428a35c4cc79563349d8238e62d is 50, key is test_row_0/A:col10/1731836505869/Put/seqid=0 2024-11-17T09:41:47,129 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#B#compaction#268 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:47,130 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/38dff5f34e3842c28f5f4ab8c249fe4a is 50, key is test_row_0/B:col10/1731836505174/Put/seqid=0 2024-11-17T09:41:47,133 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#A#compaction#269 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:47,134 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/c30128513819495eacf9f47d65e0cf98 is 50, key is test_row_0/A:col10/1731836505174/Put/seqid=0 2024-11-17T09:41:47,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742142_1318 (size=12779) 2024-11-17T09:41:47,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742141_1317 (size=12301) 2024-11-17T09:41:47,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742143_1319 (size=12779) 2024-11-17T09:41:47,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T09:41:47,543 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/4cbb5428a35c4cc79563349d8238e62d 2024-11-17T09:41:47,547 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/38dff5f34e3842c28f5f4ab8c249fe4a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38dff5f34e3842c28f5f4ab8c249fe4a 2024-11-17T09:41:47,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/450725e5532b4986a3058f33c352c51b is 50, key is test_row_0/B:col10/1731836505869/Put/seqid=0 2024-11-17T09:41:47,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742144_1320 (size=12301) 2024-11-17T09:41:47,556 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/450725e5532b4986a3058f33c352c51b 2024-11-17T09:41:47,556 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/c30128513819495eacf9f47d65e0cf98 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c30128513819495eacf9f47d65e0cf98 2024-11-17T09:41:47,557 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/B of 7ef56adb9d9541a35f7ad38e99d99e41 into 38dff5f34e3842c28f5f4ab8c249fe4a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:47,557 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:47,557 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/B, priority=8, startTime=1731836507097; duration=0sec 2024-11-17T09:41:47,557 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:47,557 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:B 2024-11-17T09:41:47,557 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-17T09:41:47,563 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/A of 7ef56adb9d9541a35f7ad38e99d99e41 into c30128513819495eacf9f47d65e0cf98(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:47,563 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:47,563 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/A, priority=8, startTime=1731836507097; duration=0sec 2024-11-17T09:41:47,563 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:47,563 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:A 2024-11-17T09:41:47,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/2803b33ff39845fb8b0690033e4f302c is 50, key is test_row_0/C:col10/1731836505869/Put/seqid=0 2024-11-17T09:41:47,566 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 95470 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-17T09:41:47,566 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/C is initiating minor compaction (all files) 2024-11-17T09:41:47,566 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/C in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:47,566 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c98bd0b28bbf4a33ba739aa04a25af52, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bcba0fb22c8b40c4b0a762ac29449fe2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/f30cc734a8fc4e2aaab124f12bd434cf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/9931f50bb8844b86bb5ef0fd69dc34fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/08d4034d160c4e3191c5224873b4791e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8c2b5f5884e34b378daa91365cc41b7a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/83e62f46183849f483dc61dd37a583b3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2ea3875c3e0c4920b78a279319bf413b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=93.2 K 2024-11-17T09:41:47,567 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c98bd0b28bbf4a33ba739aa04a25af52, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731836497475 2024-11-17T09:41:47,567 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting bcba0fb22c8b40c4b0a762ac29449fe2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731836498268 2024-11-17T09:41:47,567 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting f30cc734a8fc4e2aaab124f12bd434cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731836498303 2024-11-17T09:41:47,568 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 9931f50bb8844b86bb5ef0fd69dc34fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731836499484 2024-11-17T09:41:47,569 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 08d4034d160c4e3191c5224873b4791e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731836500659 2024-11-17T09:41:47,569 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c2b5f5884e34b378daa91365cc41b7a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731836501825 2024-11-17T09:41:47,570 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e62f46183849f483dc61dd37a583b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731836502987 2024-11-17T09:41:47,570 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea3875c3e0c4920b78a279319bf413b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731836505174 2024-11-17T09:41:47,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742145_1321 (size=12301) 2024-11-17T09:41:47,608 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/2803b33ff39845fb8b0690033e4f302c 2024-11-17T09:41:47,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/4cbb5428a35c4cc79563349d8238e62d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/4cbb5428a35c4cc79563349d8238e62d 2024-11-17T09:41:47,624 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/4cbb5428a35c4cc79563349d8238e62d, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T09:41:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/450725e5532b4986a3058f33c352c51b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/450725e5532b4986a3058f33c352c51b 2024-11-17T09:41:47,627 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#C#compaction#272 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:47,628 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/60dae011712b47439f24e6086091c282 is 50, key is test_row_0/C:col10/1731836505174/Put/seqid=0 2024-11-17T09:41:47,631 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/450725e5532b4986a3058f33c352c51b, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T09:41:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/2803b33ff39845fb8b0690033e4f302c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2803b33ff39845fb8b0690033e4f302c 2024-11-17T09:41:47,637 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2803b33ff39845fb8b0690033e4f302c, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T09:41:47,639 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 7ef56adb9d9541a35f7ad38e99d99e41 in 525ms, sequenceid=322, compaction requested=false 2024-11-17T09:41:47,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:47,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:47,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T09:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T09:41:47,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T09:41:47,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-11-17T09:41:47,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6040 sec 2024-11-17T09:41:47,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742146_1322 (size=12779) 2024-11-17T09:41:47,666 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/60dae011712b47439f24e6086091c282 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/60dae011712b47439f24e6086091c282 2024-11-17T09:41:47,675 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/C of 7ef56adb9d9541a35f7ad38e99d99e41 into 60dae011712b47439f24e6086091c282(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:47,675 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:47,675 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/C, priority=8, startTime=1731836507097; duration=0sec 2024-11-17T09:41:47,676 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:47,676 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:C 2024-11-17T09:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:48,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:41:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:48,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:48,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b4baf2826679413daa941f75c635f114 is 50, key is test_row_1/A:col10/1731836508038/Put/seqid=0 2024-11-17T09:41:48,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742147_1323 (size=14737) 2024-11-17T09:41:48,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836508058, max timestamp=1731836508071Num uniques 859; 2024-11-17T09:41:48,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836508058, max timestamp=1731836508071Num uniques 859; 2024-11-17T09:41:48,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:48,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:48,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836508058, max timestamp=1731836508071Num uniques 859; 2024-11-17T09:41:48,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:48,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836568134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836568134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836568135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T09:41:48,190 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:48,190 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T09:41:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T09:41:48,192 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:48,192 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:48,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:48,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836568250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836568250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836568250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T09:41:48,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b4baf2826679413daa941f75c635f114 2024-11-17T09:41:48,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836568460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836568460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836568460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6918350f43fd460d8e975d8d0542af24 is 50, key is test_row_1/B:col10/1731836508038/Put/seqid=0 2024-11-17T09:41:48,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742148_1324 (size=9857) 2024-11-17T09:41:48,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T09:41:48,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836568769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836568769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836568770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:48,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T09:41:48,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6918350f43fd460d8e975d8d0542af24 2024-11-17T09:41:48,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/ad944c939ae242cfbd045133186fd9f5 is 50, key is test_row_1/C:col10/1731836508038/Put/seqid=0 2024-11-17T09:41:48,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742149_1325 (size=9857) 2024-11-17T09:41:48,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:48,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:48,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:49,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:49,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:49,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:49,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:49,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:49,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/ad944c939ae242cfbd045133186fd9f5 2024-11-17T09:41:49,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836569279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:49,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836569279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:49,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836569280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:49,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:49,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b4baf2826679413daa941f75c635f114 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b4baf2826679413daa941f75c635f114 2024-11-17T09:41:49,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b4baf2826679413daa941f75c635f114, entries=200, sequenceid=335, filesize=14.4 K 2024-11-17T09:41:49,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6918350f43fd460d8e975d8d0542af24 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6918350f43fd460d8e975d8d0542af24 2024-11-17T09:41:49,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6918350f43fd460d8e975d8d0542af24, entries=100, sequenceid=335, filesize=9.6 K 2024-11-17T09:41:49,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/ad944c939ae242cfbd045133186fd9f5 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/ad944c939ae242cfbd045133186fd9f5 2024-11-17T09:41:49,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/ad944c939ae242cfbd045133186fd9f5, entries=100, sequenceid=335, filesize=9.6 K 2024-11-17T09:41:49,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1261ms, sequenceid=335, compaction requested=true 2024-11-17T09:41:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:49,300 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:49,300 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:49,301 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:49,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39817 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:49,301 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/B is initiating minor compaction (all files) 2024-11-17T09:41:49,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/A is initiating minor compaction (all files) 2024-11-17T09:41:49,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/A in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,301 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/B in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c30128513819495eacf9f47d65e0cf98, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/4cbb5428a35c4cc79563349d8238e62d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b4baf2826679413daa941f75c635f114] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=38.9 K 2024-11-17T09:41:49,301 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38dff5f34e3842c28f5f4ab8c249fe4a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/450725e5532b4986a3058f33c352c51b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6918350f43fd460d8e975d8d0542af24] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=34.1 K 2024-11-17T09:41:49,301 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 38dff5f34e3842c28f5f4ab8c249fe4a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731836505174 2024-11-17T09:41:49,302 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30128513819495eacf9f47d65e0cf98, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731836505174 2024-11-17T09:41:49,302 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 450725e5532b4986a3058f33c352c51b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731836505869 2024-11-17T09:41:49,302 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbb5428a35c4cc79563349d8238e62d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731836505869 2024-11-17T09:41:49,302 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6918350f43fd460d8e975d8d0542af24, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731836508038 2024-11-17T09:41:49,302 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4baf2826679413daa941f75c635f114, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731836508034 2024-11-17T09:41:49,308 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:49,308 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/a1ca5eda098e4a8899be6fc7b7f1cc30 is 50, key is test_row_0/B:col10/1731836505869/Put/seqid=0 2024-11-17T09:41:49,309 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:49,310 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6e815dc018224e688181e31978421442 is 50, key is test_row_0/A:col10/1731836505869/Put/seqid=0 2024-11-17T09:41:49,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742150_1326 (size=12881) 2024-11-17T09:41:49,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742151_1327 (size=12881) 2024-11-17T09:41:49,321 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/a1ca5eda098e4a8899be6fc7b7f1cc30 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a1ca5eda098e4a8899be6fc7b7f1cc30 2024-11-17T09:41:49,323 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6e815dc018224e688181e31978421442 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6e815dc018224e688181e31978421442 2024-11-17T09:41:49,326 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/B of 7ef56adb9d9541a35f7ad38e99d99e41 into a1ca5eda098e4a8899be6fc7b7f1cc30(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:49,327 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:49,327 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/B, priority=13, startTime=1731836509300; duration=0sec 2024-11-17T09:41:49,327 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:49,327 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:B 2024-11-17T09:41:49,327 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:49,328 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:49,329 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/C is initiating minor compaction (all files) 2024-11-17T09:41:49,329 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/A of 7ef56adb9d9541a35f7ad38e99d99e41 into 6e815dc018224e688181e31978421442(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:49,329 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/C in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,329 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:49,329 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/A, priority=13, startTime=1731836509300; duration=0sec 2024-11-17T09:41:49,329 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/60dae011712b47439f24e6086091c282, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2803b33ff39845fb8b0690033e4f302c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/ad944c939ae242cfbd045133186fd9f5] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=34.1 K 2024-11-17T09:41:49,329 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:49,329 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:A 2024-11-17T09:41:49,329 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 60dae011712b47439f24e6086091c282, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731836505174 2024-11-17T09:41:49,330 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2803b33ff39845fb8b0690033e4f302c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731836505869 2024-11-17T09:41:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T09:41:49,331 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ad944c939ae242cfbd045133186fd9f5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731836508038 2024-11-17T09:41:49,337 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:49,338 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/b65cf6451ac94e23a1269f1b326c5389 is 50, key is test_row_0/C:col10/1731836505869/Put/seqid=0 2024-11-17T09:41:49,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742152_1328 (size=12881) 2024-11-17T09:41:49,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T09:41:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:49,417 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T09:41:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/c195587c9aa14688adeccf3b31934d5c is 50, key is test_row_0/A:col10/1731836508102/Put/seqid=0 2024-11-17T09:41:49,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742153_1329 (size=12301) 2024-11-17T09:41:49,746 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/b65cf6451ac94e23a1269f1b326c5389 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b65cf6451ac94e23a1269f1b326c5389 2024-11-17T09:41:49,751 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/C of 7ef56adb9d9541a35f7ad38e99d99e41 into b65cf6451ac94e23a1269f1b326c5389(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:49,751 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:49,751 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/C, priority=13, startTime=1731836509300; duration=0sec 2024-11-17T09:41:49,751 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:49,751 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:C 2024-11-17T09:41:49,828 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/c195587c9aa14688adeccf3b31934d5c 2024-11-17T09:41:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/1c52bc2161a94d1290ba464c49ce7e4c is 50, key is test_row_0/B:col10/1731836508102/Put/seqid=0 2024-11-17T09:41:49,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742154_1330 (size=12301) 2024-11-17T09:41:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:50,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:50,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836570083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836570083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836570189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836570199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,239 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/1c52bc2161a94d1290ba464c49ce7e4c 2024-11-17T09:41:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/87688e5f00e64b56bee355ce3a7726c6 is 50, key is test_row_0/C:col10/1731836508102/Put/seqid=0 2024-11-17T09:41:50,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742155_1331 (size=12301) 2024-11-17T09:41:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836570289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836570289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,294 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,294 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836570300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,302 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T09:41:50,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836570400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836570410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,651 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/87688e5f00e64b56bee355ce3a7726c6 2024-11-17T09:41:50,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/c195587c9aa14688adeccf3b31934d5c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c195587c9aa14688adeccf3b31934d5c 2024-11-17T09:41:50,659 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c195587c9aa14688adeccf3b31934d5c, entries=150, sequenceid=374, filesize=12.0 K 2024-11-17T09:41:50,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/1c52bc2161a94d1290ba464c49ce7e4c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/1c52bc2161a94d1290ba464c49ce7e4c 2024-11-17T09:41:50,663 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/1c52bc2161a94d1290ba464c49ce7e4c, entries=150, sequenceid=374, filesize=12.0 K 2024-11-17T09:41:50,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/87688e5f00e64b56bee355ce3a7726c6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/87688e5f00e64b56bee355ce3a7726c6 2024-11-17T09:41:50,669 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/87688e5f00e64b56bee355ce3a7726c6, entries=150, sequenceid=374, filesize=12.0 K 2024-11-17T09:41:50,670 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1253ms, sequenceid=374, compaction requested=false 2024-11-17T09:41:50,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:50,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:50,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T09:41:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T09:41:50,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T09:41:50,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4790 sec 2024-11-17T09:41:50,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.4840 sec 2024-11-17T09:41:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:50,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:41:50,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:50,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:50,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:50,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:50,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:50,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:50,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/1d0b05fa806f452284e213b48440c0e9 is 50, key is test_row_0/A:col10/1731836510076/Put/seqid=0 2024-11-17T09:41:50,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742156_1332 (size=12301) 2024-11-17T09:41:50,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836510742, max timestamp=1731836510759Num uniques 859; 2024-11-17T09:41:50,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836510742, max timestamp=1731836510759Num uniques 859; 2024-11-17T09:41:50,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:50,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:50,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836510742, max timestamp=1731836510759Num uniques 859; 2024-11-17T09:41:50,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:41:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836510784, max timestamp=1731836510815Num uniques 1018; 2024-11-17T09:41:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836510784, max timestamp=1731836510815Num uniques 1018; 2024-11-17T09:41:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T09:41:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T09:41:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836510784, max timestamp=1731836510815Num uniques 1018; 2024-11-17T09:41:50,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T09:41:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836570889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836570890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:51,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836570999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:51,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836570999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:51,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/1d0b05fa806f452284e213b48440c0e9 2024-11-17T09:41:51,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/76dc32f56d6c4a608040b3039e9b29ed is 50, key is test_row_0/B:col10/1731836510076/Put/seqid=0 2024-11-17T09:41:51,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742157_1333 (size=12301) 2024-11-17T09:41:51,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836571210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:51,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836571210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:51,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836571519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:51,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836571519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:51,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/76dc32f56d6c4a608040b3039e9b29ed 2024-11-17T09:41:51,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/bc64e0601c42410bb1a4cc7b3c940a0b is 50, key is test_row_0/C:col10/1731836510076/Put/seqid=0 2024-11-17T09:41:51,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742158_1334 (size=12301) 2024-11-17T09:41:51,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/bc64e0601c42410bb1a4cc7b3c940a0b 2024-11-17T09:41:51,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/1d0b05fa806f452284e213b48440c0e9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/1d0b05fa806f452284e213b48440c0e9 2024-11-17T09:41:51,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/1d0b05fa806f452284e213b48440c0e9, entries=150, sequenceid=385, filesize=12.0 K 2024-11-17T09:41:51,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/76dc32f56d6c4a608040b3039e9b29ed as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/76dc32f56d6c4a608040b3039e9b29ed 2024-11-17T09:41:51,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/76dc32f56d6c4a608040b3039e9b29ed, entries=150, sequenceid=385, filesize=12.0 K 2024-11-17T09:41:51,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/bc64e0601c42410bb1a4cc7b3c940a0b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bc64e0601c42410bb1a4cc7b3c940a0b 2024-11-17T09:41:51,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bc64e0601c42410bb1a4cc7b3c940a0b, entries=150, sequenceid=385, filesize=12.0 K 2024-11-17T09:41:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1250ms, sequenceid=385, compaction requested=true 2024-11-17T09:41:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:51,962 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:51,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:51,962 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:51,963 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:51,963 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/A is initiating minor compaction (all files) 2024-11-17T09:41:51,963 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/A in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:51,963 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:51,963 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/B is initiating minor compaction (all files) 2024-11-17T09:41:51,963 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6e815dc018224e688181e31978421442, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c195587c9aa14688adeccf3b31934d5c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/1d0b05fa806f452284e213b48440c0e9] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=36.6 K 2024-11-17T09:41:51,963 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/B in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:51,963 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a1ca5eda098e4a8899be6fc7b7f1cc30, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/1c52bc2161a94d1290ba464c49ce7e4c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/76dc32f56d6c4a608040b3039e9b29ed] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=36.6 K 2024-11-17T09:41:51,964 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e815dc018224e688181e31978421442, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731836505869 2024-11-17T09:41:51,964 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ca5eda098e4a8899be6fc7b7f1cc30, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731836505869 2024-11-17T09:41:51,964 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c52bc2161a94d1290ba464c49ce7e4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731836508089 2024-11-17T09:41:51,964 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c195587c9aa14688adeccf3b31934d5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731836508089 2024-11-17T09:41:51,964 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 76dc32f56d6c4a608040b3039e9b29ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731836510076 2024-11-17T09:41:51,965 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d0b05fa806f452284e213b48440c0e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731836510076 2024-11-17T09:41:51,974 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#A#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:51,975 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:51,975 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/88cc0395b7a34e9fb19d701ce81ac7ab is 50, key is test_row_0/B:col10/1731836510076/Put/seqid=0 2024-11-17T09:41:51,976 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/03f842fa2669496a966fc50ce964b4d3 is 50, key is test_row_0/A:col10/1731836510076/Put/seqid=0 2024-11-17T09:41:51,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742159_1335 (size=12983) 2024-11-17T09:41:51,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742160_1336 (size=12983) 2024-11-17T09:41:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:52,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-17T09:41:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T09:41:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T09:41:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T09:41:52,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/f458da96dd38481ca04122fd71a79098 is 50, key is test_row_0/A:col10/1731836510853/Put/seqid=0 2024-11-17T09:41:52,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836572051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742161_1337 (size=14741) 2024-11-17T09:41:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836572053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836572160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836572160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836572300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,304 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836572309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,315 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52170 deadline: 1731836572319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,323 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T09:41:52,360 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:52,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:41:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T09:41:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T09:41:52,362 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:41:52,363 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:41:52,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:41:52,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836572370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836572370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,386 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/88cc0395b7a34e9fb19d701ce81ac7ab as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/88cc0395b7a34e9fb19d701ce81ac7ab 2024-11-17T09:41:52,390 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/B of 7ef56adb9d9541a35f7ad38e99d99e41 into 88cc0395b7a34e9fb19d701ce81ac7ab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:52,390 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:52,390 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/B, priority=13, startTime=1731836511962; duration=0sec 2024-11-17T09:41:52,390 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:52,391 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:B 2024-11-17T09:41:52,391 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:52,392 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:52,392 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/C is initiating minor compaction (all files) 2024-11-17T09:41:52,392 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/C in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,392 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b65cf6451ac94e23a1269f1b326c5389, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/87688e5f00e64b56bee355ce3a7726c6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bc64e0601c42410bb1a4cc7b3c940a0b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=36.6 K 2024-11-17T09:41:52,392 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b65cf6451ac94e23a1269f1b326c5389, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731836505869 2024-11-17T09:41:52,393 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 87688e5f00e64b56bee355ce3a7726c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731836508089 2024-11-17T09:41:52,393 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting bc64e0601c42410bb1a4cc7b3c940a0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731836510076 2024-11-17T09:41:52,393 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/03f842fa2669496a966fc50ce964b4d3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/03f842fa2669496a966fc50ce964b4d3 2024-11-17T09:41:52,398 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/A of 7ef56adb9d9541a35f7ad38e99d99e41 into 03f842fa2669496a966fc50ce964b4d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:52,398 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:52,398 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/A, priority=13, startTime=1731836511962; duration=0sec 2024-11-17T09:41:52,398 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:52,398 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:A 2024-11-17T09:41:52,400 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#C#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:52,400 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/6c7ffe9dcbf740d4877840ca1a371b0f is 50, key is test_row_0/C:col10/1731836510076/Put/seqid=0 2024-11-17T09:41:52,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742162_1338 (size=12983) 2024-11-17T09:41:52,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/f458da96dd38481ca04122fd71a79098 2024-11-17T09:41:52,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/3d404cdbc96542d2b76e5577a43d8740 is 50, key is test_row_0/B:col10/1731836510853/Put/seqid=0 2024-11-17T09:41:52,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742163_1339 (size=12301) 2024-11-17T09:41:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T09:41:52,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T09:41:52,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T09:41:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T09:41:52,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836572680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836572680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:52,809 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/6c7ffe9dcbf740d4877840ca1a371b0f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/6c7ffe9dcbf740d4877840ca1a371b0f 2024-11-17T09:41:52,814 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/C of 7ef56adb9d9541a35f7ad38e99d99e41 into 6c7ffe9dcbf740d4877840ca1a371b0f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:52,814 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:52,814 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/C, priority=13, startTime=1731836511962; duration=0sec 2024-11-17T09:41:52,814 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:52,814 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:C 2024-11-17T09:41:52,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T09:41:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/3d404cdbc96542d2b76e5577a43d8740 2024-11-17T09:41:52,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/893f431b44704273aa8180d42f9a6770 is 50, key is test_row_0/C:col10/1731836510853/Put/seqid=0 2024-11-17T09:41:52,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742164_1340 (size=12301) 2024-11-17T09:41:52,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T09:41:52,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:52,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T09:41:53,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T09:41:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:53,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:53,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:53,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836573190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:53,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836573190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:53,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T09:41:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:53,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:53,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:41:53,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/893f431b44704273aa8180d42f9a6770 2024-11-17T09:41:53,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/f458da96dd38481ca04122fd71a79098 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/f458da96dd38481ca04122fd71a79098 2024-11-17T09:41:53,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/f458da96dd38481ca04122fd71a79098, entries=200, sequenceid=440, filesize=14.4 K 2024-11-17T09:41:53,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/3d404cdbc96542d2b76e5577a43d8740 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3d404cdbc96542d2b76e5577a43d8740 2024-11-17T09:41:53,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3d404cdbc96542d2b76e5577a43d8740, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T09:41:53,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/893f431b44704273aa8180d42f9a6770 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/893f431b44704273aa8180d42f9a6770 2024-11-17T09:41:53,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/893f431b44704273aa8180d42f9a6770, entries=150, sequenceid=440, filesize=12.0 K 2024-11-17T09:41:53,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1268ms, sequenceid=440, compaction requested=false 2024-11-17T09:41:53,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:53,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T09:41:53,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:53,434 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T09:41:53,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:53,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:53,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:53,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b9cd5f88a8504ba2b5f90c31b3ed6a70 is 50, key is test_row_0/A:col10/1731836512051/Put/seqid=0 2024-11-17T09:41:53,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742165_1341 (size=9857) 2024-11-17T09:41:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T09:41:53,843 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b9cd5f88a8504ba2b5f90c31b3ed6a70 2024-11-17T09:41:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/a388e66dfb504309b5d887a101349d76 is 50, key is test_row_0/B:col10/1731836512051/Put/seqid=0 2024-11-17T09:41:53,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742166_1342 (size=9857) 2024-11-17T09:41:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:54,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. as already flushing 2024-11-17T09:41:54,256 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/a388e66dfb504309b5d887a101349d76 2024-11-17T09:41:54,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836514223, max timestamp=1731836514254Num uniques 859; 2024-11-17T09:41:54,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836514223, max timestamp=1731836514254Num uniques 859; 2024-11-17T09:41:54,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836514223, max timestamp=1731836514254Num uniques 859; 2024-11-17T09:41:54,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:54,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:54,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:54,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/c5f7703bf1a14a5ea433fbf36948b2ff is 50, key is test_row_0/C:col10/1731836512051/Put/seqid=0 2024-11-17T09:41:54,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742167_1343 (size=9857) 2024-11-17T09:41:54,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836514295, max timestamp=1731836514307Num uniques 1068; 2024-11-17T09:41:54,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836514295, max timestamp=1731836514307Num uniques 1068; 2024-11-17T09:41:54,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T09:41:54,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T09:41:54,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836514295, max timestamp=1731836514307Num uniques 1068; 2024-11-17T09:41:54,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T09:41:54,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836574364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:54,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836514331, max timestamp=1731836514367Num uniques 1077; 2024-11-17T09:41:54,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-17T09:41:54,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836514331, max timestamp=1731836514367Num uniques 1077; 2024-11-17T09:41:54,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-17T09:41:54,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836514331, max timestamp=1731836514367Num uniques 1077; 2024-11-17T09:41:54,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-17T09:41:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:54,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731836514422, max timestamp=1731836514467Num uniques 1036; 2024-11-17T09:41:54,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T09:41:54,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731836514422, max timestamp=1731836514467Num uniques 1036; 2024-11-17T09:41:54,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T09:41:54,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731836514422, max timestamp=1731836514467Num uniques 1036; 2024-11-17T09:41:54,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T09:41:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T09:41:54,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1731836514514, max timestamp=1731836514530Num uniques 1045; 2024-11-17T09:41:54,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-17T09:41:54,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1731836514514, max timestamp=1731836514530Num uniques 1045; 2024-11-17T09:41:54,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-17T09:41:54,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1731836514514, max timestamp=1731836514530Num uniques 1045; 2024-11-17T09:41:54,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47850, new segment data size=8920, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-17T09:41:54,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731836514567, max timestamp=1731836514586Num uniques 1054; 2024-11-17T09:41:54,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-17T09:41:54,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731836514567, max timestamp=1731836514586Num uniques 1054; 2024-11-17T09:41:54,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-17T09:41:54,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731836514567, max timestamp=1731836514586Num uniques 1054; 2024-11-17T09:41:54,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-17T09:41:54,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836514625, max timestamp=1731836514641Num uniques 1013; 2024-11-17T09:41:54,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-17T09:41:54,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836514625, max timestamp=1731836514641Num uniques 1013; 2024-11-17T09:41:54,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-17T09:41:54,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836514625, max timestamp=1731836514641Num uniques 1013; 2024-11-17T09:41:54,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-17T09:41:54,669 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/c5f7703bf1a14a5ea433fbf36948b2ff 2024-11-17T09:41:54,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/b9cd5f88a8504ba2b5f90c31b3ed6a70 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b9cd5f88a8504ba2b5f90c31b3ed6a70 2024-11-17T09:41:54,679 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b9cd5f88a8504ba2b5f90c31b3ed6a70, entries=100, sequenceid=451, filesize=9.6 K 2024-11-17T09:41:54,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/a388e66dfb504309b5d887a101349d76 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a388e66dfb504309b5d887a101349d76 2024-11-17T09:41:54,685 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a388e66dfb504309b5d887a101349d76, entries=100, sequenceid=451, filesize=9.6 K 2024-11-17T09:41:54,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/c5f7703bf1a14a5ea433fbf36948b2ff as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c5f7703bf1a14a5ea433fbf36948b2ff 2024-11-17T09:41:54,699 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c5f7703bf1a14a5ea433fbf36948b2ff, entries=100, sequenceid=451, filesize=9.6 K 2024-11-17T09:41:54,700 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=134.18 KB/137400 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1265ms, sequenceid=451, compaction requested=true 2024-11-17T09:41:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T09:41:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T09:41:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:54,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=140.89 KB heapSize=399.84 KB 2024-11-17T09:41:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:54,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T09:41:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:54,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-17T09:41:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:54,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.3440 sec 2024-11-17T09:41:54,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/d32c4e38c1c547f4a9dfc5e7d40e863e is 50, key is test_row_0/A:col10/1731836514701/Put/seqid=0 2024-11-17T09:41:54,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742168_1344 (size=14741) 2024-11-17T09:41:54,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/d32c4e38c1c547f4a9dfc5e7d40e863e 2024-11-17T09:41:54,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836574718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:54,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836574720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:54,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/b2e32e1a3cd0481b8dae473a3e481627 is 50, key is test_row_0/B:col10/1731836514701/Put/seqid=0 2024-11-17T09:41:54,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742169_1345 (size=12301) 2024-11-17T09:41:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836574830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836574830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:54,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836575039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836575039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:55,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/b2e32e1a3cd0481b8dae473a3e481627 2024-11-17T09:41:55,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/d09eb9afe7984d7da4f269767230fa2b is 50, key is test_row_0/C:col10/1731836514701/Put/seqid=0 2024-11-17T09:41:55,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742170_1346 (size=12301) 2024-11-17T09:41:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836575350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:55,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836575350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:55,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/d09eb9afe7984d7da4f269767230fa2b 2024-11-17T09:41:55,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/d32c4e38c1c547f4a9dfc5e7d40e863e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/d32c4e38c1c547f4a9dfc5e7d40e863e 2024-11-17T09:41:55,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/d32c4e38c1c547f4a9dfc5e7d40e863e, entries=200, sequenceid=591, filesize=14.4 K 2024-11-17T09:41:55,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/b2e32e1a3cd0481b8dae473a3e481627 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/b2e32e1a3cd0481b8dae473a3e481627 2024-11-17T09:41:55,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/b2e32e1a3cd0481b8dae473a3e481627, entries=150, sequenceid=591, filesize=12.0 K 2024-11-17T09:41:55,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/d09eb9afe7984d7da4f269767230fa2b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d09eb9afe7984d7da4f269767230fa2b 2024-11-17T09:41:55,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d09eb9afe7984d7da4f269767230fa2b, entries=150, sequenceid=591, filesize=12.0 K 2024-11-17T09:41:55,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.80 KB/409392, currentSize=40.25 KB/41220 for 7ef56adb9d9541a35f7ad38e99d99e41 in 854ms, sequenceid=591, compaction requested=true 2024-11-17T09:41:55,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:55,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:55,557 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:55,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:55,557 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:55,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:55,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:55,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:55,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:55,558 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52322 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:55,558 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/A is initiating minor compaction (all files) 2024-11-17T09:41:55,558 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/A in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:55,558 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/03f842fa2669496a966fc50ce964b4d3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/f458da96dd38481ca04122fd71a79098, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b9cd5f88a8504ba2b5f90c31b3ed6a70, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/d32c4e38c1c547f4a9dfc5e7d40e863e] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=51.1 K 2024-11-17T09:41:55,559 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:55,559 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/B is initiating minor compaction (all files) 2024-11-17T09:41:55,559 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/B in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:55,559 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/88cc0395b7a34e9fb19d701ce81ac7ab, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3d404cdbc96542d2b76e5577a43d8740, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a388e66dfb504309b5d887a101349d76, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/b2e32e1a3cd0481b8dae473a3e481627] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=46.3 K 2024-11-17T09:41:55,559 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03f842fa2669496a966fc50ce964b4d3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731836510076 2024-11-17T09:41:55,559 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 88cc0395b7a34e9fb19d701ce81ac7ab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731836510076 2024-11-17T09:41:55,559 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f458da96dd38481ca04122fd71a79098, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731836510853 2024-11-17T09:41:55,560 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9cd5f88a8504ba2b5f90c31b3ed6a70, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731836512044 2024-11-17T09:41:55,560 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d404cdbc96542d2b76e5577a43d8740, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731836510853 2024-11-17T09:41:55,560 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a388e66dfb504309b5d887a101349d76, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731836512044 2024-11-17T09:41:55,560 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32c4e38c1c547f4a9dfc5e7d40e863e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731836514632 2024-11-17T09:41:55,560 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e32e1a3cd0481b8dae473a3e481627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731836514632 2024-11-17T09:41:55,568 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#A#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:55,569 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/aa4110920616406ea248dfdae3e334e5 is 50, key is test_row_0/A:col10/1731836514701/Put/seqid=0 2024-11-17T09:41:55,569 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#B#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:55,570 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/a8e67843abe0489d81202e1129c575b6 is 50, key is test_row_0/B:col10/1731836514701/Put/seqid=0 2024-11-17T09:41:55,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742171_1347 (size=13119) 2024-11-17T09:41:55,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742172_1348 (size=13119) 2024-11-17T09:41:55,581 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/a8e67843abe0489d81202e1129c575b6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a8e67843abe0489d81202e1129c575b6 2024-11-17T09:41:55,586 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/B of 7ef56adb9d9541a35f7ad38e99d99e41 into a8e67843abe0489d81202e1129c575b6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:55,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:55,586 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/B, priority=12, startTime=1731836515557; duration=0sec 2024-11-17T09:41:55,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:55,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:B 2024-11-17T09:41:55,586 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:41:55,587 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:41:55,587 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/C is initiating minor compaction (all files) 2024-11-17T09:41:55,587 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/C in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:55,587 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/6c7ffe9dcbf740d4877840ca1a371b0f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/893f431b44704273aa8180d42f9a6770, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c5f7703bf1a14a5ea433fbf36948b2ff, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d09eb9afe7984d7da4f269767230fa2b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=46.3 K 2024-11-17T09:41:55,588 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c7ffe9dcbf740d4877840ca1a371b0f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731836510076 2024-11-17T09:41:55,588 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 893f431b44704273aa8180d42f9a6770, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731836510853 2024-11-17T09:41:55,589 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f7703bf1a14a5ea433fbf36948b2ff, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731836512044 2024-11-17T09:41:55,589 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d09eb9afe7984d7da4f269767230fa2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731836514632 2024-11-17T09:41:55,597 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:55,598 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/8f258900154548159838247c515791f3 is 50, key is test_row_0/C:col10/1731836514701/Put/seqid=0 2024-11-17T09:41:55,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742173_1349 (size=13119) 2024-11-17T09:41:55,610 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/8f258900154548159838247c515791f3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8f258900154548159838247c515791f3 2024-11-17T09:41:55,617 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/C of 7ef56adb9d9541a35f7ad38e99d99e41 into 8f258900154548159838247c515791f3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:55,617 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:55,617 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/C, priority=12, startTime=1731836515557; duration=0sec 2024-11-17T09:41:55,617 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:55,617 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:C 2024-11-17T09:41:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:55,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T09:41:55,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:55,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:55,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:55,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:55,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:55,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:41:55,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6a048da2df1d476cbf278bdff19e21fd is 50, key is test_row_0/A:col10/1731836514713/Put/seqid=0 2024-11-17T09:41:55,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742174_1350 (size=17181) 2024-11-17T09:41:55,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6a048da2df1d476cbf278bdff19e21fd 2024-11-17T09:41:55,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6b731dccd8164074ac5cf2abe382e1f3 is 50, key is test_row_0/B:col10/1731836514713/Put/seqid=0 2024-11-17T09:41:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742175_1351 (size=12301) 2024-11-17T09:41:55,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6b731dccd8164074ac5cf2abe382e1f3 2024-11-17T09:41:55,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/848b5e129cfa4d8fac49a61fcef6f946 is 50, key is test_row_0/C:col10/1731836514713/Put/seqid=0 2024-11-17T09:41:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836515902, max timestamp=1731836515926Num uniques 859; 2024-11-17T09:41:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836515902, max timestamp=1731836515926Num uniques 859; 2024-11-17T09:41:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836515902, max timestamp=1731836515926Num uniques 859; 2024-11-17T09:41:55,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:41:55,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742176_1352 (size=12301) 2024-11-17T09:41:55,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836575976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:55,983 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/aa4110920616406ea248dfdae3e334e5 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/aa4110920616406ea248dfdae3e334e5 2024-11-17T09:41:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836575982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:55,990 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/A of 7ef56adb9d9541a35f7ad38e99d99e41 into aa4110920616406ea248dfdae3e334e5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:55,990 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:55,990 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/A, priority=12, startTime=1731836515557; duration=0sec 2024-11-17T09:41:55,990 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:55,990 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:A 2024-11-17T09:41:56,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836576090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:56,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836576100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:56,175 INFO [Thread-1271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1271. 2024-11-17T09:41:56,175 DEBUG [Thread-1271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,175 DEBUG [Thread-1271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,176 DEBUG [Thread-1271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,177 INFO [Thread-1265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1265. 2024-11-17T09:41:56,177 DEBUG [Thread-1265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,177 DEBUG [Thread-1265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,178 DEBUG [Thread-1265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,180 INFO [Thread-1267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1267. 2024-11-17T09:41:56,180 DEBUG [Thread-1267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,180 DEBUG [Thread-1267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,180 DEBUG [Thread-1267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,182 INFO [Thread-1273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1273. 2024-11-17T09:41:56,182 INFO [Thread-1269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1269. 2024-11-17T09:41:56,182 DEBUG [Thread-1273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,182 DEBUG [Thread-1273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,182 DEBUG [Thread-1269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,182 DEBUG [Thread-1269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,182 DEBUG [Thread-1273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,182 DEBUG [Thread-1269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52118 deadline: 1731836576310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52180 deadline: 1731836576310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:56,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52208 deadline: 1731836576319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:56,320 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:56,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=604 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/848b5e129cfa4d8fac49a61fcef6f946 2024-11-17T09:41:56,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/6a048da2df1d476cbf278bdff19e21fd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6a048da2df1d476cbf278bdff19e21fd 2024-11-17T09:41:56,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:41:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52194 deadline: 1731836576350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 2024-11-17T09:41:56,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6a048da2df1d476cbf278bdff19e21fd, entries=250, sequenceid=604, filesize=16.8 K 2024-11-17T09:41:56,351 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 7ef56adb9d9541a35f7ad38e99d99e41 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:41:56,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/6b731dccd8164074ac5cf2abe382e1f3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6b731dccd8164074ac5cf2abe382e1f3 2024-11-17T09:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:41:56,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6b731dccd8164074ac5cf2abe382e1f3, entries=150, sequenceid=604, filesize=12.0 K 2024-11-17T09:41:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/848b5e129cfa4d8fac49a61fcef6f946 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/848b5e129cfa4d8fac49a61fcef6f946 2024-11-17T09:41:56,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/848b5e129cfa4d8fac49a61fcef6f946, entries=150, sequenceid=604, filesize=12.0 K 2024-11-17T09:41:56,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7ef56adb9d9541a35f7ad38e99d99e41 in 497ms, sequenceid=604, compaction requested=false 2024-11-17T09:41:56,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:41:56,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T09:41:56,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:41:56,372 INFO [Thread-1258 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1258. 2024-11-17T09:41:56,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:56,372 DEBUG [Thread-1258 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:41:56,372 DEBUG [Thread-1258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:56,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:41:56,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:41:56,372 DEBUG [Thread-1258 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/18f3809d8e1548d99a736785b89710a4 is 50, key is test_row_0/A:col10/1731836515980/Put/seqid=0 2024-11-17T09:41:56,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742177_1353 (size=12301) 2024-11-17T09:41:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T09:41:56,530 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:41:56,622 INFO [Thread-1254 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1254. 2024-11-17T09:41:56,622 DEBUG [Thread-1254 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,622 DEBUG [Thread-1254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,622 INFO [Thread-1260 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1260. 2024-11-17T09:41:56,622 DEBUG [Thread-1260 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:41:56,622 DEBUG [Thread-1260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,622 DEBUG [Thread-1254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,622 DEBUG [Thread-1260 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:41:56,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:41:56,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/18f3809d8e1548d99a736785b89710a4 2024-11-17T09:41:56,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/32fcad08e88747bb9d0bc0af0c93a2bc is 50, key is test_row_0/B:col10/1731836515980/Put/seqid=0 2024-11-17T09:41:56,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742178_1354 (size=12301) 2024-11-17T09:41:57,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/32fcad08e88747bb9d0bc0af0c93a2bc 2024-11-17T09:41:57,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/b09dd57df55844a3821d713a78e83322 is 50, key is test_row_0/C:col10/1731836515980/Put/seqid=0 2024-11-17T09:41:57,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742179_1355 (size=12301) 2024-11-17T09:41:57,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=641 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/b09dd57df55844a3821d713a78e83322 2024-11-17T09:41:57,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/18f3809d8e1548d99a736785b89710a4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/18f3809d8e1548d99a736785b89710a4 2024-11-17T09:41:57,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/18f3809d8e1548d99a736785b89710a4, entries=150, sequenceid=641, filesize=12.0 K 2024-11-17T09:41:57,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/32fcad08e88747bb9d0bc0af0c93a2bc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/32fcad08e88747bb9d0bc0af0c93a2bc 2024-11-17T09:41:57,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/32fcad08e88747bb9d0bc0af0c93a2bc, entries=150, sequenceid=641, filesize=12.0 K 2024-11-17T09:41:57,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/b09dd57df55844a3821d713a78e83322 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b09dd57df55844a3821d713a78e83322 2024-11-17T09:41:57,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b09dd57df55844a3821d713a78e83322, entries=150, sequenceid=641, filesize=12.0 K 2024-11-17T09:41:57,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=13.42 KB/13740 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1258ms, sequenceid=641, compaction requested=true 2024-11-17T09:41:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:41:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:41:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7ef56adb9d9541a35f7ad38e99d99e41:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:41:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:57,630 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:57,630 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:57,631 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:57,631 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/B is initiating minor compaction (all files) 2024-11-17T09:41:57,631 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:57,631 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/B in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:57,631 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/A is initiating minor compaction (all files) 2024-11-17T09:41:57,631 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a8e67843abe0489d81202e1129c575b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6b731dccd8164074ac5cf2abe382e1f3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/32fcad08e88747bb9d0bc0af0c93a2bc] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=36.8 K 2024-11-17T09:41:57,631 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/A in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:57,631 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/aa4110920616406ea248dfdae3e334e5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6a048da2df1d476cbf278bdff19e21fd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/18f3809d8e1548d99a736785b89710a4] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=41.6 K 2024-11-17T09:41:57,631 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e67843abe0489d81202e1129c575b6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731836514632 2024-11-17T09:41:57,632 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa4110920616406ea248dfdae3e334e5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731836514632 2024-11-17T09:41:57,632 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b731dccd8164074ac5cf2abe382e1f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731836514713 2024-11-17T09:41:57,632 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 32fcad08e88747bb9d0bc0af0c93a2bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731836515975 2024-11-17T09:41:57,632 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a048da2df1d476cbf278bdff19e21fd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731836514713 2024-11-17T09:41:57,632 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18f3809d8e1548d99a736785b89710a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731836515975 2024-11-17T09:41:57,639 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:57,640 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/394e6622684b43fd922df957c1678525 is 50, key is test_row_0/B:col10/1731836515980/Put/seqid=0 2024-11-17T09:41:57,648 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#A#compaction#307 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:57,648 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/9f86f6c1e9e742acb56ac7e9f3a68b8a is 50, key is test_row_0/A:col10/1731836515980/Put/seqid=0 2024-11-17T09:41:57,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742180_1356 (size=13221) 2024-11-17T09:41:57,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742181_1357 (size=13221) 2024-11-17T09:41:58,053 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/394e6622684b43fd922df957c1678525 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/394e6622684b43fd922df957c1678525 2024-11-17T09:41:58,058 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/B of 7ef56adb9d9541a35f7ad38e99d99e41 into 394e6622684b43fd922df957c1678525(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:58,058 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:58,058 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/B, priority=13, startTime=1731836517630; duration=0sec 2024-11-17T09:41:58,058 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:41:58,058 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:B 2024-11-17T09:41:58,058 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:41:58,059 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/9f86f6c1e9e742acb56ac7e9f3a68b8a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9f86f6c1e9e742acb56ac7e9f3a68b8a 2024-11-17T09:41:58,059 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:41:58,059 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 7ef56adb9d9541a35f7ad38e99d99e41/C is initiating minor compaction (all files) 2024-11-17T09:41:58,059 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7ef56adb9d9541a35f7ad38e99d99e41/C in TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:41:58,059 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8f258900154548159838247c515791f3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/848b5e129cfa4d8fac49a61fcef6f946, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b09dd57df55844a3821d713a78e83322] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp, totalSize=36.8 K 2024-11-17T09:41:58,060 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f258900154548159838247c515791f3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731836514632 2024-11-17T09:41:58,060 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 848b5e129cfa4d8fac49a61fcef6f946, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=604, earliestPutTs=1731836514713 2024-11-17T09:41:58,060 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b09dd57df55844a3821d713a78e83322, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=641, earliestPutTs=1731836515975 2024-11-17T09:41:58,063 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/A of 7ef56adb9d9541a35f7ad38e99d99e41 into 9f86f6c1e9e742acb56ac7e9f3a68b8a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:58,063 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:58,063 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/A, priority=13, startTime=1731836517630; duration=0sec 2024-11-17T09:41:58,064 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:58,064 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:A 2024-11-17T09:41:58,070 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7ef56adb9d9541a35f7ad38e99d99e41#C#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:41:58,071 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/585829e66b7c421894f1211e884992de is 50, key is test_row_0/C:col10/1731836515980/Put/seqid=0 2024-11-17T09:41:58,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742182_1358 (size=13221) 2024-11-17T09:41:58,479 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/585829e66b7c421894f1211e884992de as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/585829e66b7c421894f1211e884992de 2024-11-17T09:41:58,483 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7ef56adb9d9541a35f7ad38e99d99e41/C of 7ef56adb9d9541a35f7ad38e99d99e41 into 585829e66b7c421894f1211e884992de(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:41:58,483 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7ef56adb9d9541a35f7ad38e99d99e41: 2024-11-17T09:41:58,483 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., storeName=7ef56adb9d9541a35f7ad38e99d99e41/C, priority=13, startTime=1731836517630; duration=0sec 2024-11-17T09:41:58,483 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:41:58,483 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7ef56adb9d9541a35f7ad38e99d99e41:C 2024-11-17T09:42:00,490 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T09:42:06,371 INFO [Thread-1262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1262. 2024-11-17T09:42:06,371 DEBUG [Thread-1262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:06,371 DEBUG [Thread-1262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:06,371 DEBUG [Thread-1262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:06,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:06,441 INFO [Thread-1256 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1256. 2024-11-17T09:42:06,441 DEBUG [Thread-1256 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:06,441 DEBUG [Thread-1256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:06,441 DEBUG [Thread-1256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:06,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 163 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2654 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2654 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2797 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2641 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2644 2024-11-17T09:42:06,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T09:42:06,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:42:06,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:42:06,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:06,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:06,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:06,443 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T09:42:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T09:42:06,445 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836526445"}]},"ts":"1731836526445"} 2024-11-17T09:42:06,446 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T09:42:06,447 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T09:42:06,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:42:06,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, UNASSIGN}] 2024-11-17T09:42:06,449 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, UNASSIGN 2024-11-17T09:42:06,449 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=7ef56adb9d9541a35f7ad38e99d99e41, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:06,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, UNASSIGN because future has completed 2024-11-17T09:42:06,451 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:42:06,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:42:06,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T09:42:06,603 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 7ef56adb9d9541a35f7ad38e99d99e41, disabling compactions & flushes 2024-11-17T09:42:06,603 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. after waiting 0 ms 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:42:06,603 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 7ef56adb9d9541a35f7ad38e99d99e41 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=A 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=B 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7ef56adb9d9541a35f7ad38e99d99e41, store=C 2024-11-17T09:42:06,603 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:06,607 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/669da0e3b26a4a31ab4743e16615edd1 is 50, key is test_row_0/A:col10/1731836516620/Put/seqid=0 2024-11-17T09:42:06,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742183_1359 (size=12301) 2024-11-17T09:42:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T09:42:07,011 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=651 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/669da0e3b26a4a31ab4743e16615edd1 2024-11-17T09:42:07,017 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/ef8813d90f1a4ef89fe7a8e91aed81b4 is 50, key is test_row_0/B:col10/1731836516620/Put/seqid=0 2024-11-17T09:42:07,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742184_1360 (size=12301) 2024-11-17T09:42:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T09:42:07,420 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=651 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/ef8813d90f1a4ef89fe7a8e91aed81b4 2024-11-17T09:42:07,426 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/d22a5a8a2b124ddf95e59ddf8f9ac7fd is 50, key is test_row_0/C:col10/1731836516620/Put/seqid=0 2024-11-17T09:42:07,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742185_1361 (size=12301) 2024-11-17T09:42:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T09:42:07,829 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=651 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/d22a5a8a2b124ddf95e59ddf8f9ac7fd 2024-11-17T09:42:07,833 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/A/669da0e3b26a4a31ab4743e16615edd1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/669da0e3b26a4a31ab4743e16615edd1 2024-11-17T09:42:07,836 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/669da0e3b26a4a31ab4743e16615edd1, entries=150, sequenceid=651, filesize=12.0 K 2024-11-17T09:42:07,836 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/B/ef8813d90f1a4ef89fe7a8e91aed81b4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/ef8813d90f1a4ef89fe7a8e91aed81b4 2024-11-17T09:42:07,839 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/ef8813d90f1a4ef89fe7a8e91aed81b4, entries=150, sequenceid=651, filesize=12.0 K 2024-11-17T09:42:07,840 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/.tmp/C/d22a5a8a2b124ddf95e59ddf8f9ac7fd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d22a5a8a2b124ddf95e59ddf8f9ac7fd 2024-11-17T09:42:07,842 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d22a5a8a2b124ddf95e59ddf8f9ac7fd, entries=150, sequenceid=651, filesize=12.0 K 2024-11-17T09:42:07,843 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1240ms, sequenceid=651, compaction requested=false 2024-11-17T09:42:07,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6543ee266b1c46ceb853d8c61f6a8954, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9549c2046eb8408c98fe912da7b0c7cb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b58624f8be144b35a314b9305c1c820b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/bc316a652e4c4150a64197e9143f5d86, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/a19cb6509e2c486ab667c5f564081b90, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/5b2129c121424c64a37faedcc7a1b8cb, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b75afb465cc546069d59ca634b8a0c1b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/fdf8d39e29fd419caf9b7db4d6aacb0c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/144084db86c042dda7ba6aaad8cdf5c4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/180b93172a434d6ea221f67cbd5b6bbe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/56588685c1bb42b9931b80c99e8bf017, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/dd0951db82804b45b7d4c7914ede5879, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/7a74c38ca2d042c5a98b1ad50dd69f46, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/8bbddc08523a4cd1bae9df2ce1c6b550, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c30128513819495eacf9f47d65e0cf98, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/4cbb5428a35c4cc79563349d8238e62d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b4baf2826679413daa941f75c635f114, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6e815dc018224e688181e31978421442, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c195587c9aa14688adeccf3b31934d5c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/03f842fa2669496a966fc50ce964b4d3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/1d0b05fa806f452284e213b48440c0e9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/f458da96dd38481ca04122fd71a79098, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b9cd5f88a8504ba2b5f90c31b3ed6a70, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/d32c4e38c1c547f4a9dfc5e7d40e863e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/aa4110920616406ea248dfdae3e334e5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6a048da2df1d476cbf278bdff19e21fd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/18f3809d8e1548d99a736785b89710a4] to archive 2024-11-17T09:42:07,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:42:07,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6543ee266b1c46ceb853d8c61f6a8954 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6543ee266b1c46ceb853d8c61f6a8954 2024-11-17T09:42:07,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9549c2046eb8408c98fe912da7b0c7cb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9549c2046eb8408c98fe912da7b0c7cb 2024-11-17T09:42:07,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b58624f8be144b35a314b9305c1c820b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b58624f8be144b35a314b9305c1c820b 2024-11-17T09:42:07,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/bc316a652e4c4150a64197e9143f5d86 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/bc316a652e4c4150a64197e9143f5d86 2024-11-17T09:42:07,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/a19cb6509e2c486ab667c5f564081b90 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/a19cb6509e2c486ab667c5f564081b90 2024-11-17T09:42:07,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/5b2129c121424c64a37faedcc7a1b8cb to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/5b2129c121424c64a37faedcc7a1b8cb 2024-11-17T09:42:07,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b75afb465cc546069d59ca634b8a0c1b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b75afb465cc546069d59ca634b8a0c1b 2024-11-17T09:42:07,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/fdf8d39e29fd419caf9b7db4d6aacb0c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/fdf8d39e29fd419caf9b7db4d6aacb0c 2024-11-17T09:42:07,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/144084db86c042dda7ba6aaad8cdf5c4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/144084db86c042dda7ba6aaad8cdf5c4 2024-11-17T09:42:07,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/180b93172a434d6ea221f67cbd5b6bbe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/180b93172a434d6ea221f67cbd5b6bbe 2024-11-17T09:42:07,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/56588685c1bb42b9931b80c99e8bf017 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/56588685c1bb42b9931b80c99e8bf017 2024-11-17T09:42:07,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/dd0951db82804b45b7d4c7914ede5879 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/dd0951db82804b45b7d4c7914ede5879 2024-11-17T09:42:07,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/7a74c38ca2d042c5a98b1ad50dd69f46 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/7a74c38ca2d042c5a98b1ad50dd69f46 2024-11-17T09:42:07,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/8bbddc08523a4cd1bae9df2ce1c6b550 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/8bbddc08523a4cd1bae9df2ce1c6b550 2024-11-17T09:42:07,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c30128513819495eacf9f47d65e0cf98 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c30128513819495eacf9f47d65e0cf98 2024-11-17T09:42:07,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/4cbb5428a35c4cc79563349d8238e62d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/4cbb5428a35c4cc79563349d8238e62d 2024-11-17T09:42:07,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b4baf2826679413daa941f75c635f114 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b4baf2826679413daa941f75c635f114 2024-11-17T09:42:07,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6e815dc018224e688181e31978421442 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6e815dc018224e688181e31978421442 2024-11-17T09:42:07,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c195587c9aa14688adeccf3b31934d5c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/c195587c9aa14688adeccf3b31934d5c 2024-11-17T09:42:07,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/03f842fa2669496a966fc50ce964b4d3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/03f842fa2669496a966fc50ce964b4d3 2024-11-17T09:42:07,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/1d0b05fa806f452284e213b48440c0e9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/1d0b05fa806f452284e213b48440c0e9 2024-11-17T09:42:07,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/f458da96dd38481ca04122fd71a79098 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/f458da96dd38481ca04122fd71a79098 2024-11-17T09:42:07,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b9cd5f88a8504ba2b5f90c31b3ed6a70 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/b9cd5f88a8504ba2b5f90c31b3ed6a70 2024-11-17T09:42:07,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/d32c4e38c1c547f4a9dfc5e7d40e863e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/d32c4e38c1c547f4a9dfc5e7d40e863e 2024-11-17T09:42:07,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/aa4110920616406ea248dfdae3e334e5 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/aa4110920616406ea248dfdae3e334e5 2024-11-17T09:42:07,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6a048da2df1d476cbf278bdff19e21fd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/6a048da2df1d476cbf278bdff19e21fd 2024-11-17T09:42:07,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/18f3809d8e1548d99a736785b89710a4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/18f3809d8e1548d99a736785b89710a4 2024-11-17T09:42:07,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bf11d736dd24bb09dfd6737b6d4892e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38907c5d6e3843429437c52eb633ac52, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/5af87c0831e9420c8402ac48d6a761c4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7a5e712f740c4efba87563a045f4a173, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6d5c79fe6dd040089e61548f2c4c5535, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/9aa1869af1764387946f859d315ea0de, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7ab64349dadc45f4b797223e449d9663, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/afd5fc89dcc6494e8651ab333d3ae44f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bed662a49c346d58b72d6470e4e6b4f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/d7c1820b10c84a09a7cdaf80d454c755, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/8b4731f5eb41454f9477c1d90fe74690, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/47e07514f57d4d6aafa2a3d2b8833c38, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3b0ddb10d2ec409cb7bec5f0af80b93c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38dff5f34e3842c28f5f4ab8c249fe4a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/18a55df54442450cb65836ccb2fde13b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/450725e5532b4986a3058f33c352c51b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a1ca5eda098e4a8899be6fc7b7f1cc30, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6918350f43fd460d8e975d8d0542af24, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/1c52bc2161a94d1290ba464c49ce7e4c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/88cc0395b7a34e9fb19d701ce81ac7ab, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/76dc32f56d6c4a608040b3039e9b29ed, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3d404cdbc96542d2b76e5577a43d8740, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a388e66dfb504309b5d887a101349d76, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a8e67843abe0489d81202e1129c575b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/b2e32e1a3cd0481b8dae473a3e481627, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6b731dccd8164074ac5cf2abe382e1f3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/32fcad08e88747bb9d0bc0af0c93a2bc] to archive 2024-11-17T09:42:07,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:42:07,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bf11d736dd24bb09dfd6737b6d4892e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bf11d736dd24bb09dfd6737b6d4892e 2024-11-17T09:42:07,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38907c5d6e3843429437c52eb633ac52 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38907c5d6e3843429437c52eb633ac52 2024-11-17T09:42:07,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/5af87c0831e9420c8402ac48d6a761c4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/5af87c0831e9420c8402ac48d6a761c4 2024-11-17T09:42:07,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7a5e712f740c4efba87563a045f4a173 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7a5e712f740c4efba87563a045f4a173 2024-11-17T09:42:07,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6d5c79fe6dd040089e61548f2c4c5535 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6d5c79fe6dd040089e61548f2c4c5535 2024-11-17T09:42:07,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/9aa1869af1764387946f859d315ea0de to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/9aa1869af1764387946f859d315ea0de 2024-11-17T09:42:07,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7ab64349dadc45f4b797223e449d9663 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/7ab64349dadc45f4b797223e449d9663 2024-11-17T09:42:07,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/afd5fc89dcc6494e8651ab333d3ae44f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/afd5fc89dcc6494e8651ab333d3ae44f 2024-11-17T09:42:07,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bed662a49c346d58b72d6470e4e6b4f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/4bed662a49c346d58b72d6470e4e6b4f 2024-11-17T09:42:07,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/d7c1820b10c84a09a7cdaf80d454c755 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/d7c1820b10c84a09a7cdaf80d454c755 2024-11-17T09:42:07,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/8b4731f5eb41454f9477c1d90fe74690 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/8b4731f5eb41454f9477c1d90fe74690 2024-11-17T09:42:07,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/47e07514f57d4d6aafa2a3d2b8833c38 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/47e07514f57d4d6aafa2a3d2b8833c38 2024-11-17T09:42:07,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3b0ddb10d2ec409cb7bec5f0af80b93c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3b0ddb10d2ec409cb7bec5f0af80b93c 2024-11-17T09:42:07,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38dff5f34e3842c28f5f4ab8c249fe4a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/38dff5f34e3842c28f5f4ab8c249fe4a 2024-11-17T09:42:07,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/18a55df54442450cb65836ccb2fde13b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/18a55df54442450cb65836ccb2fde13b 2024-11-17T09:42:07,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/450725e5532b4986a3058f33c352c51b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/450725e5532b4986a3058f33c352c51b 2024-11-17T09:42:07,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a1ca5eda098e4a8899be6fc7b7f1cc30 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a1ca5eda098e4a8899be6fc7b7f1cc30 2024-11-17T09:42:07,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6918350f43fd460d8e975d8d0542af24 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6918350f43fd460d8e975d8d0542af24 2024-11-17T09:42:07,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/1c52bc2161a94d1290ba464c49ce7e4c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/1c52bc2161a94d1290ba464c49ce7e4c 2024-11-17T09:42:07,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/88cc0395b7a34e9fb19d701ce81ac7ab to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/88cc0395b7a34e9fb19d701ce81ac7ab 2024-11-17T09:42:07,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/76dc32f56d6c4a608040b3039e9b29ed to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/76dc32f56d6c4a608040b3039e9b29ed 2024-11-17T09:42:07,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3d404cdbc96542d2b76e5577a43d8740 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/3d404cdbc96542d2b76e5577a43d8740 2024-11-17T09:42:07,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a388e66dfb504309b5d887a101349d76 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a388e66dfb504309b5d887a101349d76 2024-11-17T09:42:07,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a8e67843abe0489d81202e1129c575b6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/a8e67843abe0489d81202e1129c575b6 2024-11-17T09:42:07,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/b2e32e1a3cd0481b8dae473a3e481627 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/b2e32e1a3cd0481b8dae473a3e481627 2024-11-17T09:42:07,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6b731dccd8164074ac5cf2abe382e1f3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/6b731dccd8164074ac5cf2abe382e1f3 2024-11-17T09:42:07,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/32fcad08e88747bb9d0bc0af0c93a2bc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/32fcad08e88747bb9d0bc0af0c93a2bc 2024-11-17T09:42:07,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/346d29dc64c94a20b61f173e51613f9b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/97686f99bd6a45d6b84a1315aac122c3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/e51f07f6596d4bfb854b7e7757f4e226, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/56089bb144c341d6a4d3d6a73a030c73, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/29ff9ca3f32b4626a10f03a9bc13258c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c98bd0b28bbf4a33ba739aa04a25af52, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/26c7ab6d45214d699a88e3504fef3f84, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bcba0fb22c8b40c4b0a762ac29449fe2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/f30cc734a8fc4e2aaab124f12bd434cf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/9931f50bb8844b86bb5ef0fd69dc34fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/08d4034d160c4e3191c5224873b4791e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8c2b5f5884e34b378daa91365cc41b7a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/83e62f46183849f483dc61dd37a583b3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/60dae011712b47439f24e6086091c282, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2ea3875c3e0c4920b78a279319bf413b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2803b33ff39845fb8b0690033e4f302c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b65cf6451ac94e23a1269f1b326c5389, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/ad944c939ae242cfbd045133186fd9f5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/87688e5f00e64b56bee355ce3a7726c6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/6c7ffe9dcbf740d4877840ca1a371b0f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bc64e0601c42410bb1a4cc7b3c940a0b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/893f431b44704273aa8180d42f9a6770, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c5f7703bf1a14a5ea433fbf36948b2ff, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8f258900154548159838247c515791f3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d09eb9afe7984d7da4f269767230fa2b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/848b5e129cfa4d8fac49a61fcef6f946, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b09dd57df55844a3821d713a78e83322] to archive 2024-11-17T09:42:07,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:42:07,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/346d29dc64c94a20b61f173e51613f9b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/346d29dc64c94a20b61f173e51613f9b 2024-11-17T09:42:07,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/97686f99bd6a45d6b84a1315aac122c3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/97686f99bd6a45d6b84a1315aac122c3 2024-11-17T09:42:07,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/e51f07f6596d4bfb854b7e7757f4e226 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/e51f07f6596d4bfb854b7e7757f4e226 2024-11-17T09:42:07,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/56089bb144c341d6a4d3d6a73a030c73 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/56089bb144c341d6a4d3d6a73a030c73 2024-11-17T09:42:07,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/29ff9ca3f32b4626a10f03a9bc13258c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/29ff9ca3f32b4626a10f03a9bc13258c 2024-11-17T09:42:07,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c98bd0b28bbf4a33ba739aa04a25af52 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c98bd0b28bbf4a33ba739aa04a25af52 2024-11-17T09:42:07,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/26c7ab6d45214d699a88e3504fef3f84 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/26c7ab6d45214d699a88e3504fef3f84 2024-11-17T09:42:07,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bcba0fb22c8b40c4b0a762ac29449fe2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bcba0fb22c8b40c4b0a762ac29449fe2 2024-11-17T09:42:07,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/f30cc734a8fc4e2aaab124f12bd434cf to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/f30cc734a8fc4e2aaab124f12bd434cf 2024-11-17T09:42:07,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/9931f50bb8844b86bb5ef0fd69dc34fe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/9931f50bb8844b86bb5ef0fd69dc34fe 2024-11-17T09:42:07,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/08d4034d160c4e3191c5224873b4791e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/08d4034d160c4e3191c5224873b4791e 2024-11-17T09:42:07,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8c2b5f5884e34b378daa91365cc41b7a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8c2b5f5884e34b378daa91365cc41b7a 2024-11-17T09:42:07,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/83e62f46183849f483dc61dd37a583b3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/83e62f46183849f483dc61dd37a583b3 2024-11-17T09:42:07,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/60dae011712b47439f24e6086091c282 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/60dae011712b47439f24e6086091c282 2024-11-17T09:42:07,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2ea3875c3e0c4920b78a279319bf413b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2ea3875c3e0c4920b78a279319bf413b 2024-11-17T09:42:07,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2803b33ff39845fb8b0690033e4f302c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/2803b33ff39845fb8b0690033e4f302c 2024-11-17T09:42:07,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b65cf6451ac94e23a1269f1b326c5389 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b65cf6451ac94e23a1269f1b326c5389 2024-11-17T09:42:07,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/ad944c939ae242cfbd045133186fd9f5 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/ad944c939ae242cfbd045133186fd9f5 2024-11-17T09:42:07,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/87688e5f00e64b56bee355ce3a7726c6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/87688e5f00e64b56bee355ce3a7726c6 2024-11-17T09:42:07,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/6c7ffe9dcbf740d4877840ca1a371b0f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/6c7ffe9dcbf740d4877840ca1a371b0f 2024-11-17T09:42:07,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bc64e0601c42410bb1a4cc7b3c940a0b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/bc64e0601c42410bb1a4cc7b3c940a0b 2024-11-17T09:42:07,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/893f431b44704273aa8180d42f9a6770 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/893f431b44704273aa8180d42f9a6770 2024-11-17T09:42:07,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c5f7703bf1a14a5ea433fbf36948b2ff to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/c5f7703bf1a14a5ea433fbf36948b2ff 2024-11-17T09:42:07,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8f258900154548159838247c515791f3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/8f258900154548159838247c515791f3 2024-11-17T09:42:07,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d09eb9afe7984d7da4f269767230fa2b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d09eb9afe7984d7da4f269767230fa2b 2024-11-17T09:42:07,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/848b5e129cfa4d8fac49a61fcef6f946 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/848b5e129cfa4d8fac49a61fcef6f946 2024-11-17T09:42:07,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b09dd57df55844a3821d713a78e83322 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/b09dd57df55844a3821d713a78e83322 2024-11-17T09:42:07,918 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/recovered.edits/654.seqid, newMaxSeqId=654, maxSeqId=1 2024-11-17T09:42:07,919 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. 2024-11-17T09:42:07,919 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 7ef56adb9d9541a35f7ad38e99d99e41: Waiting for close lock at 1731836526603Running coprocessor pre-close hooks at 1731836526603Disabling compacts and flushes for region at 1731836526603Disabling writes for close at 1731836526603Obtaining lock to block concurrent updates at 1731836526603Preparing flush snapshotting stores in 7ef56adb9d9541a35f7ad38e99d99e41 at 1731836526603Finished memstore snapshotting TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731836526603Flushing stores of TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41. at 1731836526604 (+1 ms)Flushing 7ef56adb9d9541a35f7ad38e99d99e41/A: creating writer at 1731836526604Flushing 7ef56adb9d9541a35f7ad38e99d99e41/A: appending metadata at 1731836526606 (+2 ms)Flushing 7ef56adb9d9541a35f7ad38e99d99e41/A: closing flushed file at 1731836526606Flushing 7ef56adb9d9541a35f7ad38e99d99e41/B: creating writer at 1731836527014 (+408 ms)Flushing 7ef56adb9d9541a35f7ad38e99d99e41/B: appending metadata at 1731836527016 (+2 ms)Flushing 7ef56adb9d9541a35f7ad38e99d99e41/B: closing flushed file at 1731836527016Flushing 7ef56adb9d9541a35f7ad38e99d99e41/C: creating writer at 1731836527423 (+407 ms)Flushing 7ef56adb9d9541a35f7ad38e99d99e41/C: appending metadata at 1731836527425 (+2 ms)Flushing 7ef56adb9d9541a35f7ad38e99d99e41/C: closing flushed file at 1731836527425Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e2ad24f: reopening flushed file at 1731836527833 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a0c8e52: reopening flushed file at 1731836527836 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d5f7c78: reopening flushed file at 1731836527839 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7ef56adb9d9541a35f7ad38e99d99e41 in 1240ms, sequenceid=651, compaction requested=false at 1731836527843 (+4 ms)Writing region close event to WAL at 1731836527916 (+73 ms)Running coprocessor post-close hooks at 1731836527918 (+2 ms)Closed at 1731836527919 (+1 ms) 2024-11-17T09:42:07,920 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:42:07,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=7ef56adb9d9541a35f7ad38e99d99e41, regionState=CLOSED 2024-11-17T09:42:07,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:42:07,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T09:42:07,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 7ef56adb9d9541a35f7ad38e99d99e41, server=70e37c475432,40671,1731836432285 in 1.4720 sec 2024-11-17T09:42:07,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-17T09:42:07,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7ef56adb9d9541a35f7ad38e99d99e41, UNASSIGN in 1.4760 sec 2024-11-17T09:42:07,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T09:42:07,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4790 sec 2024-11-17T09:42:07,929 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836527929"}]},"ts":"1731836527929"} 2024-11-17T09:42:07,930 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T09:42:07,930 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T09:42:07,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4870 sec 2024-11-17T09:42:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T09:42:08,591 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:08,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T09:42:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:08,593 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T09:42:08,593 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:08,595 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:42:08,597 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/recovered.edits] 2024-11-17T09:42:08,599 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/669da0e3b26a4a31ab4743e16615edd1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/669da0e3b26a4a31ab4743e16615edd1 2024-11-17T09:42:08,600 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9f86f6c1e9e742acb56ac7e9f3a68b8a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/A/9f86f6c1e9e742acb56ac7e9f3a68b8a 2024-11-17T09:42:08,602 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/394e6622684b43fd922df957c1678525 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/394e6622684b43fd922df957c1678525 2024-11-17T09:42:08,603 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/ef8813d90f1a4ef89fe7a8e91aed81b4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/B/ef8813d90f1a4ef89fe7a8e91aed81b4 2024-11-17T09:42:08,605 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/585829e66b7c421894f1211e884992de to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/585829e66b7c421894f1211e884992de 2024-11-17T09:42:08,606 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d22a5a8a2b124ddf95e59ddf8f9ac7fd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/C/d22a5a8a2b124ddf95e59ddf8f9ac7fd 2024-11-17T09:42:08,608 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/recovered.edits/654.seqid to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41/recovered.edits/654.seqid 2024-11-17T09:42:08,608 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/7ef56adb9d9541a35f7ad38e99d99e41 2024-11-17T09:42:08,608 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T09:42:08,610 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:08,612 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T09:42:08,616 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T09:42:08,617 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:08,617 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T09:42:08,617 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731836528617"}]},"ts":"9223372036854775807"} 2024-11-17T09:42:08,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T09:42:08,618 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7ef56adb9d9541a35f7ad38e99d99e41, NAME => 'TestAcidGuarantees,,1731836493949.7ef56adb9d9541a35f7ad38e99d99e41.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T09:42:08,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T09:42:08,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731836528619"}]},"ts":"9223372036854775807"} 2024-11-17T09:42:08,620 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T09:42:08,621 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:08,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-17T09:42:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T09:42:08,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T09:42:08,705 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:08,712 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=244 (was 246), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=367 (was 450), ProcessCount=11 (was 11), AvailableMemoryMB=2949 (was 3318) 2024-11-17T09:42:08,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=244, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=367, ProcessCount=11, AvailableMemoryMB=2948 2024-11-17T09:42:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:42:08,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:42:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:08,723 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:42:08,723 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:08,723 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-17T09:42:08,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:42:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T09:42:08,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742186_1362 (size=1061) 2024-11-17T09:42:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T09:42:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T09:42:09,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:42:09,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742187_1363 (size=53) 2024-11-17T09:42:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T09:42:09,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:42:09,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d732b91b920f21f520735b3babf0e4b7, disabling compactions & flushes 2024-11-17T09:42:09,535 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. after waiting 0 ms 2024-11-17T09:42:09,535 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,535 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,536 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d732b91b920f21f520735b3babf0e4b7: Waiting for close lock at 1731836529535Disabling compacts and flushes for region at 1731836529535Disabling writes for close at 1731836529535Writing region close event to WAL at 1731836529535Closed at 1731836529535 2024-11-17T09:42:09,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:42:09,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731836529537"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836529537"}]},"ts":"1731836529537"} 2024-11-17T09:42:09,538 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:42:09,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:42:09,539 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836529539"}]},"ts":"1731836529539"} 2024-11-17T09:42:09,540 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T09:42:09,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, ASSIGN}] 2024-11-17T09:42:09,541 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, ASSIGN 2024-11-17T09:42:09,542 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, ASSIGN; state=OFFLINE, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=false 2024-11-17T09:42:09,692 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:09,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, ASSIGN because future has completed 2024-11-17T09:42:09,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:42:09,849 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,849 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:42:09,849 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,849 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:42:09,849 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,849 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,850 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,851 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:09,852 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d732b91b920f21f520735b3babf0e4b7 columnFamilyName A 2024-11-17T09:42:09,852 DEBUG [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:09,852 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(327): Store=d732b91b920f21f520735b3babf0e4b7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:09,852 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,853 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:09,853 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d732b91b920f21f520735b3babf0e4b7 columnFamilyName B 2024-11-17T09:42:09,853 DEBUG [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:09,853 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(327): Store=d732b91b920f21f520735b3babf0e4b7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:09,854 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,854 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:09,854 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d732b91b920f21f520735b3babf0e4b7 columnFamilyName C 2024-11-17T09:42:09,854 DEBUG [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:09,855 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(327): Store=d732b91b920f21f520735b3babf0e4b7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:09,855 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,855 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,855 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,856 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,856 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,856 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,857 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:42:09,858 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,859 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:42:09,860 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened d732b91b920f21f520735b3babf0e4b7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74255726, jitterRate=0.10649654269218445}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:42:09,860 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T09:42:09,860 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for d732b91b920f21f520735b3babf0e4b7: Running coprocessor pre-open hook at 1731836529849Writing region info on filesystem at 1731836529849Initializing all the Stores at 1731836529850 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836529850Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836529850Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836529850Cleaning up temporary data from old regions at 1731836529856 (+6 ms)Running coprocessor post-open hooks at 1731836529860 (+4 ms)Region opened successfully at 1731836529860 2024-11-17T09:42:09,861 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., pid=87, masterSystemTime=1731836529846 2024-11-17T09:42:09,862 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,863 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:09,863 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:09,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:42:09,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T09:42:09,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 in 171 msec 2024-11-17T09:42:09,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-17T09:42:09,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, ASSIGN in 327 msec 2024-11-17T09:42:09,869 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:42:09,869 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836529869"}]},"ts":"1731836529869"} 2024-11-17T09:42:09,870 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T09:42:09,871 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:42:09,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-17T09:42:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-17T09:42:10,870 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:10,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:10,871 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:10,871 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:10,872 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c4a4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:10,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:10,874 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:10,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633d91ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:10,875 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:10,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:10,877 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:42:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,44533,1731836431485 2024-11-17T09:42:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ab9244e 2024-11-17T09:42:10,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:42:10,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:42:10,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:42:10,880 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:42:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742188_1364 (size=1097) 2024-11-17T09:42:11,291 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T09:42:11,291 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T09:42:11,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:42:11,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, REOPEN/MOVE}] 2024-11-17T09:42:11,296 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, REOPEN/MOVE 2024-11-17T09:42:11,297 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, REOPEN/MOVE because future has completed 2024-11-17T09:42:11,299 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:42:11,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:42:11,451 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,451 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:42:11,451 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing d732b91b920f21f520735b3babf0e4b7, disabling compactions & flushes 2024-11-17T09:42:11,451 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,451 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,451 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. after waiting 0 ms 2024-11-17T09:42:11,451 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,454 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T09:42:11,455 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,455 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for d732b91b920f21f520735b3babf0e4b7: Waiting for close lock at 1731836531451Running coprocessor pre-close hooks at 1731836531451Disabling compacts and flushes for region at 1731836531451Disabling writes for close at 1731836531451Writing region close event to WAL at 1731836531451Running coprocessor post-close hooks at 1731836531455 (+4 ms)Closed at 1731836531455 2024-11-17T09:42:11,455 WARN [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: d732b91b920f21f520735b3babf0e4b7 to self. 2024-11-17T09:42:11,456 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=CLOSED 2024-11-17T09:42:11,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:42:11,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T09:42:11,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 in 160 msec 2024-11-17T09:42:11,460 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, REOPEN/MOVE; state=CLOSED, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=true 2024-11-17T09:42:11,611 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, REOPEN/MOVE because future has completed 2024-11-17T09:42:11,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:42:11,767 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,767 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:42:11,767 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,767 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:42:11,767 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,767 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,768 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,769 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:11,770 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d732b91b920f21f520735b3babf0e4b7 columnFamilyName A 2024-11-17T09:42:11,770 DEBUG [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:11,771 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(327): Store=d732b91b920f21f520735b3babf0e4b7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:11,771 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,772 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:11,772 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d732b91b920f21f520735b3babf0e4b7 columnFamilyName B 2024-11-17T09:42:11,772 DEBUG [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:11,772 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(327): Store=d732b91b920f21f520735b3babf0e4b7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:11,772 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,773 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:11,773 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d732b91b920f21f520735b3babf0e4b7 columnFamilyName C 2024-11-17T09:42:11,773 DEBUG [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:11,773 INFO [StoreOpener-d732b91b920f21f520735b3babf0e4b7-1 {}] regionserver.HStore(327): Store=d732b91b920f21f520735b3babf0e4b7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:11,773 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,773 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,774 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,775 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,775 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,775 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,776 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:42:11,777 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,777 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened d732b91b920f21f520735b3babf0e4b7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63411683, jitterRate=-0.05509229004383087}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:42:11,777 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,778 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for d732b91b920f21f520735b3babf0e4b7: Running coprocessor pre-open hook at 1731836531767Writing region info on filesystem at 1731836531767Initializing all the Stores at 1731836531768 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836531768Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836531768Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836531768Cleaning up temporary data from old regions at 1731836531775 (+7 ms)Running coprocessor post-open hooks at 1731836531777 (+2 ms)Region opened successfully at 1731836531778 (+1 ms) 2024-11-17T09:42:11,779 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., pid=92, masterSystemTime=1731836531764 2024-11-17T09:42:11,780 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,780 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,781 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=OPEN, openSeqNum=5, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:42:11,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-17T09:42:11,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 in 170 msec 2024-11-17T09:42:11,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-17T09:42:11,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, REOPEN/MOVE in 490 msec 2024-11-17T09:42:11,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T09:42:11,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-17T09:42:11,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-17T09:42:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T09:42:11,791 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:11,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,793 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ac3af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,794 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,794 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71dc51c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,796 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b1b943c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3322257d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,799 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a49e27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f0f8de8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a04b52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203c0e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621aa877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,812 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f69444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,814 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac6721d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,815 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269f4659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,817 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa6ca21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,819 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8c36ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,821 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1edef8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,823 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@620f3405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11641e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,825 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7aa41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:11,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:11,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:11,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e11975b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:11,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:11,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:11,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9536c66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:11,829 DEBUG [Thread-1668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,829 DEBUG [Thread-1657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,830 DEBUG [Thread-1660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,830 DEBUG [Thread-1662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,830 DEBUG [Thread-1666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,830 DEBUG [Thread-1649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,830 DEBUG [Thread-1664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,831 DEBUG [Thread-1655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,833 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:11,833 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-17T09:42:11,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T09:42:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,837 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,838 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:11,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:11,839 DEBUG [Thread-1651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,841 DEBUG [Thread-1653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:11,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:11,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:42:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:11,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:42:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:11,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:11,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eacd82fb936848a590795ce11b0a5148_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836531840/Put/seqid=0 2024-11-17T09:42:11,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742189_1365 (size=14594) 2024-11-17T09:42:11,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836591898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836591900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836591901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836591902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836591909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:11,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T09:42:11,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:11,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836592020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836592020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836592020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836592020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836592020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T09:42:12,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836592230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836592230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836592230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836592230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836592230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:12,294 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eacd82fb936848a590795ce11b0a5148_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eacd82fb936848a590795ce11b0a5148_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:12,295 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/c21810699d4b49e7adc682b398f41abc, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:12,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/c21810699d4b49e7adc682b398f41abc is 175, key is test_row_0/A:col10/1731836531840/Put/seqid=0 2024-11-17T09:42:12,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742190_1366 (size=39549) 2024-11-17T09:42:12,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T09:42:12,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836592540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836592540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836592540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836592540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836592540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:12,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:12,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:12,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:12,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/c21810699d4b49e7adc682b398f41abc 2024-11-17T09:42:12,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5e6740145fa04508a308b26e4fdca383 is 50, key is test_row_0/B:col10/1731836531840/Put/seqid=0 2024-11-17T09:42:12,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742191_1367 (size=12001) 2024-11-17T09:42:12,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:12,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:12,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:12,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:12,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:12,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T09:42:13,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836593050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:13,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836593050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:13,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836593059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836593060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:13,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836593060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:13,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5e6740145fa04508a308b26e4fdca383 2024-11-17T09:42:13,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/5ac02e7653364011ab8d5e33671ba96f is 50, key is test_row_0/C:col10/1731836531840/Put/seqid=0 2024-11-17T09:42:13,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742192_1368 (size=12001) 2024-11-17T09:42:13,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:13,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:13,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:42:13,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:13,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/5ac02e7653364011ab8d5e33671ba96f 2024-11-17T09:42:13,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/c21810699d4b49e7adc682b398f41abc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/c21810699d4b49e7adc682b398f41abc 2024-11-17T09:42:13,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/c21810699d4b49e7adc682b398f41abc, entries=200, sequenceid=16, filesize=38.6 K 2024-11-17T09:42:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5e6740145fa04508a308b26e4fdca383 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5e6740145fa04508a308b26e4fdca383 2024-11-17T09:42:13,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5e6740145fa04508a308b26e4fdca383, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T09:42:13,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/5ac02e7653364011ab8d5e33671ba96f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5ac02e7653364011ab8d5e33671ba96f 2024-11-17T09:42:13,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5ac02e7653364011ab8d5e33671ba96f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T09:42:13,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d732b91b920f21f520735b3babf0e4b7 in 1721ms, sequenceid=16, compaction requested=false 2024-11-17T09:42:13,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:13,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-17T09:42:13,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:13,677 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:42:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:13,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175988cbb9881b45f3bcc9736df0425a69_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836531907/Put/seqid=0 2024-11-17T09:42:13,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742193_1369 (size=12154) 2024-11-17T09:42:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:13,700 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175988cbb9881b45f3bcc9736df0425a69_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175988cbb9881b45f3bcc9736df0425a69_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:13,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/4a82bba351ac4b1594204fe0d56fbf2b, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:13,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/4a82bba351ac4b1594204fe0d56fbf2b is 175, key is test_row_0/A:col10/1731836531907/Put/seqid=0 2024-11-17T09:42:13,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742194_1370 (size=30955) 2024-11-17T09:42:13,716 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/4a82bba351ac4b1594204fe0d56fbf2b 2024-11-17T09:42:13,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b0c6c731d9ef4e28be9c4c5dc36edfb4 is 50, key is test_row_0/B:col10/1731836531907/Put/seqid=0 2024-11-17T09:42:13,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742195_1371 (size=12001) 2024-11-17T09:42:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T09:42:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:14,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:14,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836594084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836594085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836594087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836594088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836594089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,138 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b0c6c731d9ef4e28be9c4c5dc36edfb4 2024-11-17T09:42:14,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/21a29e3afcb140fcb3149b1e54b281a1 is 50, key is test_row_0/C:col10/1731836531907/Put/seqid=0 2024-11-17T09:42:14,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742196_1372 (size=12001) 2024-11-17T09:42:14,149 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/21a29e3afcb140fcb3149b1e54b281a1 2024-11-17T09:42:14,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/4a82bba351ac4b1594204fe0d56fbf2b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4a82bba351ac4b1594204fe0d56fbf2b 2024-11-17T09:42:14,156 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4a82bba351ac4b1594204fe0d56fbf2b, entries=150, sequenceid=36, filesize=30.2 K 2024-11-17T09:42:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b0c6c731d9ef4e28be9c4c5dc36edfb4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b0c6c731d9ef4e28be9c4c5dc36edfb4 2024-11-17T09:42:14,161 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b0c6c731d9ef4e28be9c4c5dc36edfb4, entries=150, sequenceid=36, filesize=11.7 K 2024-11-17T09:42:14,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/21a29e3afcb140fcb3149b1e54b281a1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/21a29e3afcb140fcb3149b1e54b281a1 2024-11-17T09:42:14,165 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/21a29e3afcb140fcb3149b1e54b281a1, entries=150, sequenceid=36, filesize=11.7 K 2024-11-17T09:42:14,165 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d732b91b920f21f520735b3babf0e4b7 in 489ms, sequenceid=36, compaction requested=false 2024-11-17T09:42:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:14,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-17T09:42:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-17T09:42:14,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-17T09:42:14,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3290 sec 2024-11-17T09:42:14,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3360 sec 2024-11-17T09:42:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:14,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T09:42:14,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:14,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:14,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:14,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:14,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:14,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:14,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3c1d3023acc40a09ee17421070f149a_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836534078/Put/seqid=0 2024-11-17T09:42:14,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742197_1373 (size=14594) 2024-11-17T09:42:14,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836594234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836594235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836594236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836594246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836594247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836594360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836594360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836594360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836594370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836594370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836594580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836594580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836594580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836594589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836594589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:14,621 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f3c1d3023acc40a09ee17421070f149a_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3c1d3023acc40a09ee17421070f149a_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:14,622 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3d25eba39eac411eb7e8db5eaed27892, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:14,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3d25eba39eac411eb7e8db5eaed27892 is 175, key is test_row_0/A:col10/1731836534078/Put/seqid=0 2024-11-17T09:42:14,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742198_1374 (size=39549) 2024-11-17T09:42:14,628 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3d25eba39eac411eb7e8db5eaed27892 2024-11-17T09:42:14,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/bc4e8c5e5f0b403291fcc0c949ae7668 is 50, key is test_row_0/B:col10/1731836534078/Put/seqid=0 2024-11-17T09:42:14,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742199_1375 (size=12001) 2024-11-17T09:42:14,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836594900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836594900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836594900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836594912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:14,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836594916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:14,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:15,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/bc4e8c5e5f0b403291fcc0c949ae7668 2024-11-17T09:42:15,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/29e37c07e3754ffe975405a70f1d59a5 is 50, key is test_row_0/C:col10/1731836534078/Put/seqid=0 2024-11-17T09:42:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742200_1376 (size=12001) 2024-11-17T09:42:15,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836595420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:15,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:15,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836595420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836595430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:15,443 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836595430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836595430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:15,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/29e37c07e3754ffe975405a70f1d59a5 2024-11-17T09:42:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3d25eba39eac411eb7e8db5eaed27892 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3d25eba39eac411eb7e8db5eaed27892 2024-11-17T09:42:15,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3d25eba39eac411eb7e8db5eaed27892, entries=200, sequenceid=49, filesize=38.6 K 2024-11-17T09:42:15,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/bc4e8c5e5f0b403291fcc0c949ae7668 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/bc4e8c5e5f0b403291fcc0c949ae7668 2024-11-17T09:42:15,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/bc4e8c5e5f0b403291fcc0c949ae7668, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T09:42:15,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/29e37c07e3754ffe975405a70f1d59a5 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/29e37c07e3754ffe975405a70f1d59a5 2024-11-17T09:42:15,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/29e37c07e3754ffe975405a70f1d59a5, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T09:42:15,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d732b91b920f21f520735b3babf0e4b7 in 1307ms, sequenceid=49, compaction requested=true 2024-11-17T09:42:15,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:15,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:15,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:15,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:15,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:42:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:15,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-17T09:42:15,513 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:15,513 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:15,514 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:15,514 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/C is initiating minor compaction (all files) 2024-11-17T09:42:15,514 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/C in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:15,514 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5ac02e7653364011ab8d5e33671ba96f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/21a29e3afcb140fcb3149b1e54b281a1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/29e37c07e3754ffe975405a70f1d59a5] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.2 K 2024-11-17T09:42:15,515 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:15,515 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/A is initiating minor compaction (all files) 2024-11-17T09:42:15,515 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/A in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:15,515 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/c21810699d4b49e7adc682b398f41abc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4a82bba351ac4b1594204fe0d56fbf2b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3d25eba39eac411eb7e8db5eaed27892] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=107.5 K 2024-11-17T09:42:15,515 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:15,515 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/c21810699d4b49e7adc682b398f41abc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4a82bba351ac4b1594204fe0d56fbf2b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3d25eba39eac411eb7e8db5eaed27892] 2024-11-17T09:42:15,515 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac02e7653364011ab8d5e33671ba96f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731836531840 2024-11-17T09:42:15,516 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c21810699d4b49e7adc682b398f41abc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731836531840 2024-11-17T09:42:15,516 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a82bba351ac4b1594204fe0d56fbf2b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836531883 2024-11-17T09:42:15,516 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a29e3afcb140fcb3149b1e54b281a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836531883 2024-11-17T09:42:15,517 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d25eba39eac411eb7e8db5eaed27892, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731836534078 2024-11-17T09:42:15,517 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 29e37c07e3754ffe975405a70f1d59a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731836534078 2024-11-17T09:42:15,535 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:15,535 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/f6d1eae22370475eb4fc06a30bc6cf6d is 50, key is test_row_0/C:col10/1731836534078/Put/seqid=0 2024-11-17T09:42:15,538 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:15,549 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117529ee9ab12044a4cb57d2c5eb28c8d45_d732b91b920f21f520735b3babf0e4b7 store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:15,552 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117529ee9ab12044a4cb57d2c5eb28c8d45_d732b91b920f21f520735b3babf0e4b7, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:15,552 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117529ee9ab12044a4cb57d2c5eb28c8d45_d732b91b920f21f520735b3babf0e4b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:15,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742201_1377 (size=12104) 2024-11-17T09:42:15,605 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/f6d1eae22370475eb4fc06a30bc6cf6d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/f6d1eae22370475eb4fc06a30bc6cf6d 2024-11-17T09:42:15,610 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/C of d732b91b920f21f520735b3babf0e4b7 into f6d1eae22370475eb4fc06a30bc6cf6d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:15,610 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:15,610 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/C, priority=13, startTime=1731836535512; duration=0sec 2024-11-17T09:42:15,610 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:15,610 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:C 2024-11-17T09:42:15,610 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:15,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742202_1378 (size=4469) 2024-11-17T09:42:15,611 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:15,611 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/B is initiating minor compaction (all files) 2024-11-17T09:42:15,611 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/B in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:15,611 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5e6740145fa04508a308b26e4fdca383, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b0c6c731d9ef4e28be9c4c5dc36edfb4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/bc4e8c5e5f0b403291fcc0c949ae7668] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.2 K 2024-11-17T09:42:15,612 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e6740145fa04508a308b26e4fdca383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731836531840 2024-11-17T09:42:15,612 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b0c6c731d9ef4e28be9c4c5dc36edfb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836531883 2024-11-17T09:42:15,613 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4e8c5e5f0b403291fcc0c949ae7668, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731836534078 2024-11-17T09:42:15,626 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:15,626 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/ced8717298af4cd294491aa8d0c28d12 is 50, key is test_row_0/B:col10/1731836534078/Put/seqid=0 2024-11-17T09:42:15,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742203_1379 (size=12104) 2024-11-17T09:42:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T09:42:15,990 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:15,991 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-17T09:42:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T09:42:15,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:15,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:15,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:16,012 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#A#compaction#322 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:16,013 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/b2fa22e943a141288cd844f8ceedee7d is 175, key is test_row_0/A:col10/1731836534078/Put/seqid=0 2024-11-17T09:42:16,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742204_1380 (size=31058) 2024-11-17T09:42:16,041 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/ced8717298af4cd294491aa8d0c28d12 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/ced8717298af4cd294491aa8d0c28d12 2024-11-17T09:42:16,047 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/B of d732b91b920f21f520735b3babf0e4b7 into ced8717298af4cd294491aa8d0c28d12(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:16,047 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:16,047 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/B, priority=13, startTime=1731836535512; duration=0sec 2024-11-17T09:42:16,047 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:16,047 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:B 2024-11-17T09:42:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T09:42:16,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-17T09:42:16,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:16,147 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T09:42:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:16,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e8bf3e49ea6b45dd8cc74a161722962b_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836534246/Put/seqid=0 2024-11-17T09:42:16,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742205_1381 (size=12154) 2024-11-17T09:42:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T09:42:16,424 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/b2fa22e943a141288cd844f8ceedee7d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/b2fa22e943a141288cd844f8ceedee7d 2024-11-17T09:42:16,431 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/A of d732b91b920f21f520735b3babf0e4b7 into b2fa22e943a141288cd844f8ceedee7d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:16,431 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:16,431 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/A, priority=13, startTime=1731836535512; duration=0sec 2024-11-17T09:42:16,431 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:16,431 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:A 2024-11-17T09:42:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:16,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836596476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836596480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836596474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836596491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836596492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836596600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836596600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836596600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836596600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836596600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T09:42:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:16,636 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e8bf3e49ea6b45dd8cc74a161722962b_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e8bf3e49ea6b45dd8cc74a161722962b_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:16,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/50ca44a75c954a93bad42fa7f463fae6, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:16,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/50ca44a75c954a93bad42fa7f463fae6 is 175, key is test_row_0/A:col10/1731836534246/Put/seqid=0 2024-11-17T09:42:16,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742206_1382 (size=30955) 2024-11-17T09:42:16,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836596820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836596820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836596820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836596820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:16,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836596820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:16,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,077 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/50ca44a75c954a93bad42fa7f463fae6 2024-11-17T09:42:17,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/0582e420d2a345a6990d2d6da0cfc8ea is 50, key is test_row_0/B:col10/1731836534246/Put/seqid=0 2024-11-17T09:42:17,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742207_1383 (size=12001) 2024-11-17T09:42:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T09:42:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836597140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836597140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836597140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836597140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836597140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,505 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/0582e420d2a345a6990d2d6da0cfc8ea 2024-11-17T09:42:17,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/522bb5ab4cdd4f16b2a8f86c220baf8a is 50, key is test_row_0/C:col10/1731836534246/Put/seqid=0 2024-11-17T09:42:17,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742208_1384 (size=12001) 2024-11-17T09:42:17,608 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/522bb5ab4cdd4f16b2a8f86c220baf8a 2024-11-17T09:42:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/50ca44a75c954a93bad42fa7f463fae6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/50ca44a75c954a93bad42fa7f463fae6 2024-11-17T09:42:17,625 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/50ca44a75c954a93bad42fa7f463fae6, entries=150, sequenceid=70, filesize=30.2 K 2024-11-17T09:42:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/0582e420d2a345a6990d2d6da0cfc8ea as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/0582e420d2a345a6990d2d6da0cfc8ea 2024-11-17T09:42:17,639 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/0582e420d2a345a6990d2d6da0cfc8ea, entries=150, sequenceid=70, filesize=11.7 K 2024-11-17T09:42:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/522bb5ab4cdd4f16b2a8f86c220baf8a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/522bb5ab4cdd4f16b2a8f86c220baf8a 2024-11-17T09:42:17,650 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/522bb5ab4cdd4f16b2a8f86c220baf8a, entries=150, sequenceid=70, filesize=11.7 K 2024-11-17T09:42:17,652 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for d732b91b920f21f520735b3babf0e4b7 in 1506ms, sequenceid=70, compaction requested=false 2024-11-17T09:42:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-17T09:42:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-17T09:42:17,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-17T09:42:17,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6600 sec 2024-11-17T09:42:17,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6660 sec 2024-11-17T09:42:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:17,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T09:42:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:17,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:17,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:17,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:17,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111729ac117aaf7044b9add37fd29d363442_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836536461/Put/seqid=0 2024-11-17T09:42:17,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742209_1385 (size=14594) 2024-11-17T09:42:17,690 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:17,695 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111729ac117aaf7044b9add37fd29d363442_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111729ac117aaf7044b9add37fd29d363442_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:17,697 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3c13a6a11886470992587b7f1f77e14b, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:17,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3c13a6a11886470992587b7f1f77e14b is 175, key is test_row_0/A:col10/1731836536461/Put/seqid=0 2024-11-17T09:42:17,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836597700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836597703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836597705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836597707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836597715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742210_1386 (size=39549) 2024-11-17T09:42:17,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3c13a6a11886470992587b7f1f77e14b 2024-11-17T09:42:17,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5a7488738140469bb0b18f5dce235eca is 50, key is test_row_0/B:col10/1731836536461/Put/seqid=0 2024-11-17T09:42:17,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742211_1387 (size=12001) 2024-11-17T09:42:17,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5a7488738140469bb0b18f5dce235eca 2024-11-17T09:42:17,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/b0152bf4c49a45988574c2c1868ccd86 is 50, key is test_row_0/C:col10/1731836536461/Put/seqid=0 2024-11-17T09:42:17,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742212_1388 (size=12001) 2024-11-17T09:42:17,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836597831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836598050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836598050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836598050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836598050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836598050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-17T09:42:18,140 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:18,141 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-17T09:42:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T09:42:18,142 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:18,143 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:18,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:18,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/b0152bf4c49a45988574c2c1868ccd86 2024-11-17T09:42:18,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/3c13a6a11886470992587b7f1f77e14b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3c13a6a11886470992587b7f1f77e14b 2024-11-17T09:42:18,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3c13a6a11886470992587b7f1f77e14b, entries=200, sequenceid=86, filesize=38.6 K 2024-11-17T09:42:18,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5a7488738140469bb0b18f5dce235eca as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5a7488738140469bb0b18f5dce235eca 2024-11-17T09:42:18,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5a7488738140469bb0b18f5dce235eca, entries=150, sequenceid=86, filesize=11.7 K 2024-11-17T09:42:18,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/b0152bf4c49a45988574c2c1868ccd86 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/b0152bf4c49a45988574c2c1868ccd86 2024-11-17T09:42:18,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/b0152bf4c49a45988574c2c1868ccd86, entries=150, sequenceid=86, filesize=11.7 K 2024-11-17T09:42:18,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d732b91b920f21f520735b3babf0e4b7 in 547ms, sequenceid=86, compaction requested=true 2024-11-17T09:42:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:18,212 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:18,212 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:18,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:18,214 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:18,214 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:18,214 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/B is initiating minor compaction (all files) 2024-11-17T09:42:18,214 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/A is initiating minor compaction (all files) 2024-11-17T09:42:18,214 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/B in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:18,215 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/A in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:18,215 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/ced8717298af4cd294491aa8d0c28d12, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/0582e420d2a345a6990d2d6da0cfc8ea, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5a7488738140469bb0b18f5dce235eca] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.3 K 2024-11-17T09:42:18,215 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/b2fa22e943a141288cd844f8ceedee7d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/50ca44a75c954a93bad42fa7f463fae6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3c13a6a11886470992587b7f1f77e14b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=99.2 K 2024-11-17T09:42:18,215 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:18,215 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/b2fa22e943a141288cd844f8ceedee7d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/50ca44a75c954a93bad42fa7f463fae6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3c13a6a11886470992587b7f1f77e14b] 2024-11-17T09:42:18,215 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ced8717298af4cd294491aa8d0c28d12, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731836534078 2024-11-17T09:42:18,215 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2fa22e943a141288cd844f8ceedee7d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731836534078 2024-11-17T09:42:18,216 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50ca44a75c954a93bad42fa7f463fae6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731836534232 2024-11-17T09:42:18,216 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0582e420d2a345a6990d2d6da0cfc8ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731836534232 2024-11-17T09:42:18,217 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c13a6a11886470992587b7f1f77e14b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731836536451 2024-11-17T09:42:18,217 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a7488738140469bb0b18f5dce235eca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731836536451 2024-11-17T09:42:18,224 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:18,225 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/15f3de55c3b3420897b551fe883bd13c is 50, key is test_row_0/B:col10/1731836536461/Put/seqid=0 2024-11-17T09:42:18,240 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:18,242 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411175d1a94dac0414f1a94a55deddb35f69b_d732b91b920f21f520735b3babf0e4b7 store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:18,244 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411175d1a94dac0414f1a94a55deddb35f69b_d732b91b920f21f520735b3babf0e4b7, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:18,245 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175d1a94dac0414f1a94a55deddb35f69b_d732b91b920f21f520735b3babf0e4b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:18,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742213_1389 (size=12207) 2024-11-17T09:42:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T09:42:18,253 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/15f3de55c3b3420897b551fe883bd13c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/15f3de55c3b3420897b551fe883bd13c 2024-11-17T09:42:18,262 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/B of d732b91b920f21f520735b3babf0e4b7 into 15f3de55c3b3420897b551fe883bd13c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:18,262 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:18,262 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/B, priority=13, startTime=1731836538212; duration=0sec 2024-11-17T09:42:18,262 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:18,264 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:B 2024-11-17T09:42:18,264 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:18,265 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:18,265 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/C is initiating minor compaction (all files) 2024-11-17T09:42:18,265 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/C in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:18,265 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/f6d1eae22370475eb4fc06a30bc6cf6d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/522bb5ab4cdd4f16b2a8f86c220baf8a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/b0152bf4c49a45988574c2c1868ccd86] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.3 K 2024-11-17T09:42:18,265 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting f6d1eae22370475eb4fc06a30bc6cf6d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731836534078 2024-11-17T09:42:18,266 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 522bb5ab4cdd4f16b2a8f86c220baf8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731836534232 2024-11-17T09:42:18,266 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b0152bf4c49a45988574c2c1868ccd86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731836536451 2024-11-17T09:42:18,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742214_1390 (size=4469) 2024-11-17T09:42:18,273 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#A#compaction#331 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:18,273 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/93d694dff5fb4443ad3455ac790e4e88 is 175, key is test_row_0/A:col10/1731836536461/Put/seqid=0 2024-11-17T09:42:18,282 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:18,282 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/bbccb5bdd2db4c11a55ece38dbc6e8e8 is 50, key is test_row_0/C:col10/1731836536461/Put/seqid=0 2024-11-17T09:42:18,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742215_1391 (size=31161) 2024-11-17T09:42:18,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742216_1392 (size=12207) 2024-11-17T09:42:18,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-17T09:42:18,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:18,296 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:42:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:18,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:18,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117098d390ea8304797b7a715ef5dcd6200_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836537712/Put/seqid=0 2024-11-17T09:42:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742217_1393 (size=12154) 2024-11-17T09:42:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:18,310 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117098d390ea8304797b7a715ef5dcd6200_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117098d390ea8304797b7a715ef5dcd6200_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:18,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/66a868f388384c78aec04727674a9d61, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:18,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/66a868f388384c78aec04727674a9d61 is 175, key is test_row_0/A:col10/1731836537712/Put/seqid=0 2024-11-17T09:42:18,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742218_1394 (size=30955) 2024-11-17T09:42:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:18,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836598389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836598390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836598391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836598397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836598399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T09:42:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836598510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836598510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836598510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836598510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836598520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,696 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/bbccb5bdd2db4c11a55ece38dbc6e8e8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/bbccb5bdd2db4c11a55ece38dbc6e8e8 2024-11-17T09:42:18,697 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/93d694dff5fb4443ad3455ac790e4e88 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/93d694dff5fb4443ad3455ac790e4e88 2024-11-17T09:42:18,700 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/C of d732b91b920f21f520735b3babf0e4b7 into bbccb5bdd2db4c11a55ece38dbc6e8e8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:18,700 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/A of d732b91b920f21f520735b3babf0e4b7 into 93d694dff5fb4443ad3455ac790e4e88(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:18,700 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:18,700 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/C, priority=13, startTime=1731836538212; duration=0sec 2024-11-17T09:42:18,700 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:18,700 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/A, priority=13, startTime=1731836538212; duration=0sec 2024-11-17T09:42:18,700 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:18,700 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:C 2024-11-17T09:42:18,700 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:18,700 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:A 2024-11-17T09:42:18,717 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/66a868f388384c78aec04727674a9d61 2024-11-17T09:42:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/c33c356a4886472ba73aa94c8d74e366 is 50, key is test_row_0/B:col10/1731836537712/Put/seqid=0 2024-11-17T09:42:18,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742219_1395 (size=12001) 2024-11-17T09:42:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836598729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836598729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836598730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836598730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836598730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T09:42:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836599039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836599050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836599050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836599050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836599050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,129 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/c33c356a4886472ba73aa94c8d74e366 2024-11-17T09:42:19,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/dbbae89a4f9d411ba775a046942b2fa7 is 50, key is test_row_0/C:col10/1731836537712/Put/seqid=0 2024-11-17T09:42:19,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742220_1396 (size=12001) 2024-11-17T09:42:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T09:42:19,541 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/dbbae89a4f9d411ba775a046942b2fa7 2024-11-17T09:42:19,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/66a868f388384c78aec04727674a9d61 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/66a868f388384c78aec04727674a9d61 2024-11-17T09:42:19,550 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/66a868f388384c78aec04727674a9d61, entries=150, sequenceid=104, filesize=30.2 K 2024-11-17T09:42:19,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/c33c356a4886472ba73aa94c8d74e366 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c33c356a4886472ba73aa94c8d74e366 2024-11-17T09:42:19,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836599551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,560 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c33c356a4886472ba73aa94c8d74e366, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T09:42:19,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/dbbae89a4f9d411ba775a046942b2fa7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/dbbae89a4f9d411ba775a046942b2fa7 2024-11-17T09:42:19,565 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/dbbae89a4f9d411ba775a046942b2fa7, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T09:42:19,566 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d732b91b920f21f520735b3babf0e4b7 in 1270ms, sequenceid=104, compaction requested=false 2024-11-17T09:42:19,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:19,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:19,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-17T09:42:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-17T09:42:19,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-17T09:42:19,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-17T09:42:19,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.4290 sec 2024-11-17T09:42:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:19,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:19,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:19,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:19,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dc0b1ace366441819ea118cdb3e5727b_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836538379/Put/seqid=0 2024-11-17T09:42:19,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742221_1397 (size=14594) 2024-11-17T09:42:19,591 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:19,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dc0b1ace366441819ea118cdb3e5727b_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dc0b1ace366441819ea118cdb3e5727b_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:19,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/d388cc3c3f1f49fa8d793867fd2aa741, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:19,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/d388cc3c3f1f49fa8d793867fd2aa741 is 175, key is test_row_0/A:col10/1731836538379/Put/seqid=0 2024-11-17T09:42:19,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742222_1398 (size=39549) 2024-11-17T09:42:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836599625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836599625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836599626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836599626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836599750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836599750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836599750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836599750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836599970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836599970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836599970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836599970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:19,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,000 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/d388cc3c3f1f49fa8d793867fd2aa741 2024-11-17T09:42:20,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/c8d388fed2e9401381beea7397603f47 is 50, key is test_row_0/B:col10/1731836538379/Put/seqid=0 2024-11-17T09:42:20,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742223_1399 (size=12001) 2024-11-17T09:42:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-17T09:42:20,290 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:20,290 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-17T09:42:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T09:42:20,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:20,293 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:20,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836600290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836600290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836600290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836600290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T09:42:20,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/c8d388fed2e9401381beea7397603f47 2024-11-17T09:42:20,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/8437ef218e83446aa6784bcebb5c2869 is 50, key is test_row_0/C:col10/1731836538379/Put/seqid=0 2024-11-17T09:42:20,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742224_1400 (size=12001) 2024-11-17T09:42:20,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T09:42:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836600569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,578 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T09:42:20,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:20,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T09:42:20,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T09:42:20,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:20,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:20,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836600809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836600809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836600809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836600810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:20,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/8437ef218e83446aa6784bcebb5c2869 2024-11-17T09:42:20,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/d388cc3c3f1f49fa8d793867fd2aa741 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/d388cc3c3f1f49fa8d793867fd2aa741 2024-11-17T09:42:20,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/d388cc3c3f1f49fa8d793867fd2aa741, entries=200, sequenceid=122, filesize=38.6 K 2024-11-17T09:42:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/c8d388fed2e9401381beea7397603f47 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c8d388fed2e9401381beea7397603f47 2024-11-17T09:42:20,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c8d388fed2e9401381beea7397603f47, entries=150, sequenceid=122, filesize=11.7 K 2024-11-17T09:42:20,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/8437ef218e83446aa6784bcebb5c2869 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/8437ef218e83446aa6784bcebb5c2869 2024-11-17T09:42:20,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/8437ef218e83446aa6784bcebb5c2869, entries=150, sequenceid=122, filesize=11.7 K 2024-11-17T09:42:20,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d732b91b920f21f520735b3babf0e4b7 in 1268ms, sequenceid=122, compaction requested=true 2024-11-17T09:42:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:20,845 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:20,845 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:20,845 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:20,845 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:20,846 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/A is initiating minor compaction (all files) 2024-11-17T09:42:20,846 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/B is initiating minor compaction (all files) 2024-11-17T09:42:20,846 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/B in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,846 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/A in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,846 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/15f3de55c3b3420897b551fe883bd13c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c33c356a4886472ba73aa94c8d74e366, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c8d388fed2e9401381beea7397603f47] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.4 K 2024-11-17T09:42:20,846 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/93d694dff5fb4443ad3455ac790e4e88, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/66a868f388384c78aec04727674a9d61, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/d388cc3c3f1f49fa8d793867fd2aa741] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=99.3 K 2024-11-17T09:42:20,846 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,846 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/93d694dff5fb4443ad3455ac790e4e88, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/66a868f388384c78aec04727674a9d61, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/d388cc3c3f1f49fa8d793867fd2aa741] 2024-11-17T09:42:20,846 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d694dff5fb4443ad3455ac790e4e88, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731836536451 2024-11-17T09:42:20,847 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 15f3de55c3b3420897b551fe883bd13c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731836536451 2024-11-17T09:42:20,847 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a868f388384c78aec04727674a9d61, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731836537681 2024-11-17T09:42:20,847 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c33c356a4886472ba73aa94c8d74e366, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731836537681 2024-11-17T09:42:20,847 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d388cc3c3f1f49fa8d793867fd2aa741, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731836538379 2024-11-17T09:42:20,848 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c8d388fed2e9401381beea7397603f47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731836538379 2024-11-17T09:42:20,853 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:20,855 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c1b218f6f9f34d4eb8ee7b8fbbf7bb78_d732b91b920f21f520735b3babf0e4b7 store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:20,855 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:20,856 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/a324a7734e5d4f3f8caa04db56818c01 is 50, key is test_row_0/B:col10/1731836538379/Put/seqid=0 2024-11-17T09:42:20,857 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c1b218f6f9f34d4eb8ee7b8fbbf7bb78_d732b91b920f21f520735b3babf0e4b7, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:20,857 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c1b218f6f9f34d4eb8ee7b8fbbf7bb78_d732b91b920f21f520735b3babf0e4b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:20,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742225_1401 (size=12309) 2024-11-17T09:42:20,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742226_1402 (size=4469) 2024-11-17T09:42:20,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T09:42:20,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:20,905 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:20,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:20,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172fda10f8ed2f4606bb3b889f6c200a93_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836539624/Put/seqid=0 2024-11-17T09:42:20,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742227_1403 (size=12304) 2024-11-17T09:42:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T09:42:21,265 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/a324a7734e5d4f3f8caa04db56818c01 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a324a7734e5d4f3f8caa04db56818c01 2024-11-17T09:42:21,267 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:21,268 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/4c745f315fdd4f58869d5e88d27186fd is 175, key is test_row_0/A:col10/1731836538379/Put/seqid=0 2024-11-17T09:42:21,271 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/B of d732b91b920f21f520735b3babf0e4b7 into a324a7734e5d4f3f8caa04db56818c01(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:21,271 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:21,272 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/B, priority=13, startTime=1731836540845; duration=0sec 2024-11-17T09:42:21,272 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:21,272 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:B 2024-11-17T09:42:21,272 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:21,272 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:21,273 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/C is initiating minor compaction (all files) 2024-11-17T09:42:21,273 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/C in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:21,273 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/bbccb5bdd2db4c11a55ece38dbc6e8e8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/dbbae89a4f9d411ba775a046942b2fa7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/8437ef218e83446aa6784bcebb5c2869] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.4 K 2024-11-17T09:42:21,273 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting bbccb5bdd2db4c11a55ece38dbc6e8e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731836536451 2024-11-17T09:42:21,274 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting dbbae89a4f9d411ba775a046942b2fa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731836537681 2024-11-17T09:42:21,274 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8437ef218e83446aa6784bcebb5c2869, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731836538379 2024-11-17T09:42:21,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742228_1404 (size=31263) 2024-11-17T09:42:21,290 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:21,291 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/ee4dca1872ae4f18b5db5224ff7f882c is 50, key is test_row_0/C:col10/1731836538379/Put/seqid=0 2024-11-17T09:42:21,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742229_1405 (size=12309) 2024-11-17T09:42:21,307 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/ee4dca1872ae4f18b5db5224ff7f882c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/ee4dca1872ae4f18b5db5224ff7f882c 2024-11-17T09:42:21,311 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/C of d732b91b920f21f520735b3babf0e4b7 into ee4dca1872ae4f18b5db5224ff7f882c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:21,311 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:21,311 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/C, priority=13, startTime=1731836540845; duration=0sec 2024-11-17T09:42:21,311 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:21,311 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:C 2024-11-17T09:42:21,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:21,321 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172fda10f8ed2f4606bb3b889f6c200a93_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172fda10f8ed2f4606bb3b889f6c200a93_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:21,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/75371503b2a7475282ec54bfa072df60, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/75371503b2a7475282ec54bfa072df60 is 175, key is test_row_0/A:col10/1731836539624/Put/seqid=0 2024-11-17T09:42:21,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742230_1406 (size=31105) 2024-11-17T09:42:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T09:42:21,680 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/4c745f315fdd4f58869d5e88d27186fd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4c745f315fdd4f58869d5e88d27186fd 2024-11-17T09:42:21,692 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/A of d732b91b920f21f520735b3babf0e4b7 into 4c745f315fdd4f58869d5e88d27186fd(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:21,692 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:21,692 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/A, priority=13, startTime=1731836540844; duration=0sec 2024-11-17T09:42:21,692 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:21,692 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:A 2024-11-17T09:42:21,727 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/75371503b2a7475282ec54bfa072df60 2024-11-17T09:42:21,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/672965e351284fb3b1436a8adf619939 is 50, key is test_row_0/B:col10/1731836539624/Put/seqid=0 2024-11-17T09:42:21,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742231_1407 (size=12151) 2024-11-17T09:42:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:21,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:21,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836601850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836601858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836601860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:21,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836601861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836601969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:21,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836601979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836601980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836601980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,141 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/672965e351284fb3b1436a8adf619939 2024-11-17T09:42:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/d87e5be750a14b0392cb7e1bcc39f6e8 is 50, key is test_row_0/C:col10/1731836539624/Put/seqid=0 2024-11-17T09:42:22,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742232_1408 (size=12151) 2024-11-17T09:42:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836602189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836602199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836602199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836602200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T09:42:22,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836602500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836602520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836602520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836602520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,552 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/d87e5be750a14b0392cb7e1bcc39f6e8 2024-11-17T09:42:22,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/75371503b2a7475282ec54bfa072df60 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/75371503b2a7475282ec54bfa072df60 2024-11-17T09:42:22,559 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/75371503b2a7475282ec54bfa072df60, entries=150, sequenceid=138, filesize=30.4 K 2024-11-17T09:42:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/672965e351284fb3b1436a8adf619939 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/672965e351284fb3b1436a8adf619939 2024-11-17T09:42:22,563 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/672965e351284fb3b1436a8adf619939, entries=150, sequenceid=138, filesize=11.9 K 2024-11-17T09:42:22,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/d87e5be750a14b0392cb7e1bcc39f6e8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d87e5be750a14b0392cb7e1bcc39f6e8 2024-11-17T09:42:22,567 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d87e5be750a14b0392cb7e1bcc39f6e8, entries=150, sequenceid=138, filesize=11.9 K 2024-11-17T09:42:22,568 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d732b91b920f21f520735b3babf0e4b7 in 1663ms, sequenceid=138, compaction requested=false 2024-11-17T09:42:22,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:22,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:22,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-17T09:42:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-17T09:42:22,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-17T09:42:22,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2760 sec 2024-11-17T09:42:22,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.2810 sec 2024-11-17T09:42:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:22,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:42:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:22,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:22,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117660c697acb2746cda1d888f7b4c2cfb6_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836541858/Put/seqid=0 2024-11-17T09:42:22,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742233_1409 (size=14794) 2024-11-17T09:42:22,606 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:22,610 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117660c697acb2746cda1d888f7b4c2cfb6_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117660c697acb2746cda1d888f7b4c2cfb6_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:22,610 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/cc1bbfc607ed4ab3842b937fad593a53, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:22,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/cc1bbfc607ed4ab3842b937fad593a53 is 175, key is test_row_0/A:col10/1731836541858/Put/seqid=0 2024-11-17T09:42:22,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742234_1410 (size=39749) 2024-11-17T09:42:22,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836602700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:22,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836602820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:23,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836603009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:23,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/cc1bbfc607ed4ab3842b937fad593a53 2024-11-17T09:42:23,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b4027e81fe3f4bf58a16676553c1ce3e is 50, key is test_row_0/B:col10/1731836541858/Put/seqid=0 2024-11-17T09:42:23,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742235_1411 (size=12151) 2024-11-17T09:42:23,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836603030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836603040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836603040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836603040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:23,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836603350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:23,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b4027e81fe3f4bf58a16676553c1ce3e 2024-11-17T09:42:23,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/a0201b61f7ce4cdb9c6f23c4374cc360 is 50, key is test_row_0/C:col10/1731836541858/Put/seqid=0 2024-11-17T09:42:23,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742236_1412 (size=12151) 2024-11-17T09:42:23,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/a0201b61f7ce4cdb9c6f23c4374cc360 2024-11-17T09:42:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/cc1bbfc607ed4ab3842b937fad593a53 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/cc1bbfc607ed4ab3842b937fad593a53 2024-11-17T09:42:23,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/cc1bbfc607ed4ab3842b937fad593a53, entries=200, sequenceid=158, filesize=38.8 K 2024-11-17T09:42:23,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b4027e81fe3f4bf58a16676553c1ce3e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4027e81fe3f4bf58a16676553c1ce3e 2024-11-17T09:42:23,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4027e81fe3f4bf58a16676553c1ce3e, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T09:42:23,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/a0201b61f7ce4cdb9c6f23c4374cc360 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/a0201b61f7ce4cdb9c6f23c4374cc360 2024-11-17T09:42:23,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/a0201b61f7ce4cdb9c6f23c4374cc360, entries=150, sequenceid=158, filesize=11.9 K 2024-11-17T09:42:23,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d732b91b920f21f520735b3babf0e4b7 in 1268ms, sequenceid=158, compaction requested=true 2024-11-17T09:42:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:23,864 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:23,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:23,864 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/A is initiating minor compaction (all files) 2024-11-17T09:42:23,865 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/A in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/B is initiating minor compaction (all files) 2024-11-17T09:42:23,865 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/B in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:23,865 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4c745f315fdd4f58869d5e88d27186fd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/75371503b2a7475282ec54bfa072df60, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/cc1bbfc607ed4ab3842b937fad593a53] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=99.7 K 2024-11-17T09:42:23,865 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4c745f315fdd4f58869d5e88d27186fd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/75371503b2a7475282ec54bfa072df60, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/cc1bbfc607ed4ab3842b937fad593a53] 2024-11-17T09:42:23,865 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a324a7734e5d4f3f8caa04db56818c01, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/672965e351284fb3b1436a8adf619939, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4027e81fe3f4bf58a16676553c1ce3e] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.8 K 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c745f315fdd4f58869d5e88d27186fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731836538379 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a324a7734e5d4f3f8caa04db56818c01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731836538379 2024-11-17T09:42:23,865 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75371503b2a7475282ec54bfa072df60, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731836539604 2024-11-17T09:42:23,866 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 672965e351284fb3b1436a8adf619939, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731836539604 2024-11-17T09:42:23,866 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1bbfc607ed4ab3842b937fad593a53, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836541845 2024-11-17T09:42:23,866 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b4027e81fe3f4bf58a16676553c1ce3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836541858 2024-11-17T09:42:23,871 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:23,873 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411171fbf0aed0c4e4cc1a415e358f798cdae_d732b91b920f21f520735b3babf0e4b7 store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:23,873 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:23,874 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/a5c4dcbd86d446128ca76884764ccacd is 50, key is test_row_0/B:col10/1731836541858/Put/seqid=0 2024-11-17T09:42:23,874 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411171fbf0aed0c4e4cc1a415e358f798cdae_d732b91b920f21f520735b3babf0e4b7, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:23,874 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171fbf0aed0c4e4cc1a415e358f798cdae_d732b91b920f21f520735b3babf0e4b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:23,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742237_1413 (size=12561) 2024-11-17T09:42:23,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:23,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:23,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:23,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:23,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:23,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:23,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:23,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:23,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742238_1414 (size=4469) 2024-11-17T09:42:23,885 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/a5c4dcbd86d446128ca76884764ccacd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a5c4dcbd86d446128ca76884764ccacd 2024-11-17T09:42:23,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c58badd44d4e4afb91c6cf3758ebbdff_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836543878/Put/seqid=0 2024-11-17T09:42:23,892 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/B of d732b91b920f21f520735b3babf0e4b7 into a5c4dcbd86d446128ca76884764ccacd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:23,892 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:23,892 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/B, priority=13, startTime=1731836543864; duration=0sec 2024-11-17T09:42:23,892 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:23,892 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:B 2024-11-17T09:42:23,892 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:23,893 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:23,893 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/C is initiating minor compaction (all files) 2024-11-17T09:42:23,893 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/C in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:23,893 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/ee4dca1872ae4f18b5db5224ff7f882c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d87e5be750a14b0392cb7e1bcc39f6e8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/a0201b61f7ce4cdb9c6f23c4374cc360] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=35.8 K 2024-11-17T09:42:23,893 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ee4dca1872ae4f18b5db5224ff7f882c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731836538379 2024-11-17T09:42:23,894 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d87e5be750a14b0392cb7e1bcc39f6e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731836539604 2024-11-17T09:42:23,894 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a0201b61f7ce4cdb9c6f23c4374cc360, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836541858 2024-11-17T09:42:23,901 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:23,902 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/098fc995a8094a13adfda9619aef9581 is 50, key is test_row_0/C:col10/1731836541858/Put/seqid=0 2024-11-17T09:42:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742239_1415 (size=14794) 2024-11-17T09:42:23,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742240_1416 (size=12561) 2024-11-17T09:42:23,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c58badd44d4e4afb91c6cf3758ebbdff_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c58badd44d4e4afb91c6cf3758ebbdff_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:23,911 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/23720bcc0f6c4507957162368e68d661, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:23,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/23720bcc0f6c4507957162368e68d661 is 175, key is test_row_0/A:col10/1731836543878/Put/seqid=0 2024-11-17T09:42:23,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742241_1417 (size=39749) 2024-11-17T09:42:23,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/23720bcc0f6c4507957162368e68d661 2024-11-17T09:42:23,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b4ed47d8ed6248be9c246ae8428072db is 50, key is test_row_0/B:col10/1731836543878/Put/seqid=0 2024-11-17T09:42:23,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742242_1418 (size=12151) 2024-11-17T09:42:23,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b4ed47d8ed6248be9c246ae8428072db 2024-11-17T09:42:23,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/95c5ab17c26d44039f25602616082554 is 50, key is test_row_0/C:col10/1731836543878/Put/seqid=0 2024-11-17T09:42:23,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742243_1419 (size=12151) 2024-11-17T09:42:24,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836604006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836604040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,051 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836604060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836604060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,069 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:24,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836604060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,070 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,070 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836604120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,281 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:24,281 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/e5d8aea1ec3e4599aeaeea37dcbaadd2 is 175, key is test_row_0/A:col10/1731836541858/Put/seqid=0 2024-11-17T09:42:24,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742244_1420 (size=31515) 2024-11-17T09:42:24,297 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/e5d8aea1ec3e4599aeaeea37dcbaadd2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e5d8aea1ec3e4599aeaeea37dcbaadd2 2024-11-17T09:42:24,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/A of d732b91b920f21f520735b3babf0e4b7 into e5d8aea1ec3e4599aeaeea37dcbaadd2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:24,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:24,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/A, priority=13, startTime=1731836543864; duration=0sec 2024-11-17T09:42:24,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:24,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:A 2024-11-17T09:42:24,311 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/098fc995a8094a13adfda9619aef9581 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/098fc995a8094a13adfda9619aef9581 2024-11-17T09:42:24,315 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/C of d732b91b920f21f520735b3babf0e4b7 into 098fc995a8094a13adfda9619aef9581(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:24,315 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:24,315 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/C, priority=13, startTime=1731836543864; duration=0sec 2024-11-17T09:42:24,315 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:24,315 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:C 2024-11-17T09:42:24,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/95c5ab17c26d44039f25602616082554 2024-11-17T09:42:24,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/23720bcc0f6c4507957162368e68d661 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/23720bcc0f6c4507957162368e68d661 2024-11-17T09:42:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836604339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/23720bcc0f6c4507957162368e68d661, entries=200, sequenceid=174, filesize=38.8 K 2024-11-17T09:42:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/b4ed47d8ed6248be9c246ae8428072db as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4ed47d8ed6248be9c246ae8428072db 2024-11-17T09:42:24,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4ed47d8ed6248be9c246ae8428072db, entries=150, sequenceid=174, filesize=11.9 K 2024-11-17T09:42:24,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/95c5ab17c26d44039f25602616082554 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95c5ab17c26d44039f25602616082554 2024-11-17T09:42:24,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95c5ab17c26d44039f25602616082554, entries=150, sequenceid=174, filesize=11.9 K 2024-11-17T09:42:24,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d732b91b920f21f520735b3babf0e4b7 in 482ms, sequenceid=174, compaction requested=false 2024-11-17T09:42:24,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T09:42:24,460 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:24,460 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-17T09:42:24,462 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T09:42:24,463 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:24,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:24,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T09:42:24,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T09:42:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:24,616 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:24,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:24,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a37736aa123246e6b37fefefd9beac34_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836543990/Put/seqid=0 2024-11-17T09:42:24,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742245_1421 (size=12304) 2024-11-17T09:42:24,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:24,631 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a37736aa123246e6b37fefefd9beac34_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a37736aa123246e6b37fefefd9beac34_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:24,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8e2119135b26479788f5c05dfc62d199, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:24,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8e2119135b26479788f5c05dfc62d199 is 175, key is test_row_0/A:col10/1731836543990/Put/seqid=0 2024-11-17T09:42:24,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742246_1422 (size=31105) 2024-11-17T09:42:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:24,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:24,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836604746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T09:42:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836604860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:24,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:25,038 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8e2119135b26479788f5c05dfc62d199 2024-11-17T09:42:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/6c71a8a4bb1343eda2503996eb1a18ed is 50, key is test_row_0/B:col10/1731836543990/Put/seqid=0 2024-11-17T09:42:25,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742247_1423 (size=12151) 2024-11-17T09:42:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836605079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:25,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T09:42:25,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836605400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:25,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:25,451 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/6c71a8a4bb1343eda2503996eb1a18ed 2024-11-17T09:42:25,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/d8a8573d694c447b805a79f394248f6a is 50, key is test_row_0/C:col10/1731836543990/Put/seqid=0 2024-11-17T09:42:25,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742248_1424 (size=12151) 2024-11-17T09:42:25,470 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/d8a8573d694c447b805a79f394248f6a 2024-11-17T09:42:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8e2119135b26479788f5c05dfc62d199 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8e2119135b26479788f5c05dfc62d199 2024-11-17T09:42:25,477 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8e2119135b26479788f5c05dfc62d199, entries=150, sequenceid=193, filesize=30.4 K 2024-11-17T09:42:25,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/6c71a8a4bb1343eda2503996eb1a18ed as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/6c71a8a4bb1343eda2503996eb1a18ed 2024-11-17T09:42:25,481 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/6c71a8a4bb1343eda2503996eb1a18ed, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T09:42:25,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/d8a8573d694c447b805a79f394248f6a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d8a8573d694c447b805a79f394248f6a 2024-11-17T09:42:25,487 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d8a8573d694c447b805a79f394248f6a, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T09:42:25,488 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d732b91b920f21f520735b3babf0e4b7 in 872ms, sequenceid=193, compaction requested=true 2024-11-17T09:42:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:25,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-17T09:42:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-17T09:42:25,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T09:42:25,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0270 sec 2024-11-17T09:42:25,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.0320 sec 2024-11-17T09:42:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T09:42:25,600 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:25,600 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-17T09:42:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T09:42:25,602 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:25,602 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:25,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T09:42:25,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T09:42:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:25,755 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:25,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:25,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d5305619c8f546d19a97cc7f790a3ccb_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836544704/Put/seqid=0 2024-11-17T09:42:25,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742249_1425 (size=12304) 2024-11-17T09:42:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:25,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T09:42:26,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836606037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836606069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,081 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836606090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836606090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,100 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836606090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,101 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,101 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836606150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:26,169 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d5305619c8f546d19a97cc7f790a3ccb_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d5305619c8f546d19a97cc7f790a3ccb_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:26,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/22d44af61ff64a0c815d0fd811b4ad34, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:26,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/22d44af61ff64a0c815d0fd811b4ad34 is 175, key is test_row_0/A:col10/1731836544704/Put/seqid=0 2024-11-17T09:42:26,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742250_1426 (size=31105) 2024-11-17T09:42:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T09:42:26,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836606369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,575 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/22d44af61ff64a0c815d0fd811b4ad34 2024-11-17T09:42:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/80d3e12c064147438b5f1c9fb6f51933 is 50, key is test_row_0/B:col10/1731836544704/Put/seqid=0 2024-11-17T09:42:26,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742251_1427 (size=12151) 2024-11-17T09:42:26,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836606690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T09:42:26,985 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/80d3e12c064147438b5f1c9fb6f51933 2024-11-17T09:42:26,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/3007d3d33a6c43c38aac2b44913c0796 is 50, key is test_row_0/C:col10/1731836544704/Put/seqid=0 2024-11-17T09:42:26,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742252_1428 (size=12151) 2024-11-17T09:42:27,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836607210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:27,396 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/3007d3d33a6c43c38aac2b44913c0796 2024-11-17T09:42:27,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/22d44af61ff64a0c815d0fd811b4ad34 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/22d44af61ff64a0c815d0fd811b4ad34 2024-11-17T09:42:27,403 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/22d44af61ff64a0c815d0fd811b4ad34, entries=150, sequenceid=209, filesize=30.4 K 2024-11-17T09:42:27,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/80d3e12c064147438b5f1c9fb6f51933 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/80d3e12c064147438b5f1c9fb6f51933 2024-11-17T09:42:27,407 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/80d3e12c064147438b5f1c9fb6f51933, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T09:42:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/3007d3d33a6c43c38aac2b44913c0796 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3007d3d33a6c43c38aac2b44913c0796 2024-11-17T09:42:27,412 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3007d3d33a6c43c38aac2b44913c0796, entries=150, sequenceid=209, filesize=11.9 K 2024-11-17T09:42:27,412 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d732b91b920f21f520735b3babf0e4b7 in 1657ms, sequenceid=209, compaction requested=true 2024-11-17T09:42:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-17T09:42:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-17T09:42:27,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-17T09:42:27,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-11-17T09:42:27,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8150 sec 2024-11-17T09:42:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T09:42:27,750 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:27,750 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-17T09:42:27,753 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T09:42:27,754 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:27,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T09:42:27,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T09:42:27,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:27,909 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:27,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:27,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:27,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:27,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:27,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d92b76b3b46a4b4cafb22108a16d8b72_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836546020/Put/seqid=0 2024-11-17T09:42:27,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742253_1429 (size=12304) 2024-11-17T09:42:27,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:27,986 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d92b76b3b46a4b4cafb22108a16d8b72_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d92b76b3b46a4b4cafb22108a16d8b72_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:27,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/48199c90367a4ff389c056eecb8da5f2, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:27,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/48199c90367a4ff389c056eecb8da5f2 is 175, key is test_row_0/A:col10/1731836546020/Put/seqid=0 2024-11-17T09:42:28,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742254_1430 (size=31105) 2024-11-17T09:42:28,020 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/48199c90367a4ff389c056eecb8da5f2 2024-11-17T09:42:28,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5d71b623fc1349438090e0ef34ee2cb3 is 50, key is test_row_0/B:col10/1731836546020/Put/seqid=0 2024-11-17T09:42:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T09:42:28,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742255_1431 (size=12151) 2024-11-17T09:42:28,079 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5d71b623fc1349438090e0ef34ee2cb3 2024-11-17T09:42:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/2ee335fab38247c1bfb0aa04fe341be2 is 50, key is test_row_0/C:col10/1731836546020/Put/seqid=0 2024-11-17T09:42:28,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742256_1432 (size=12151) 2024-11-17T09:42:28,143 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/2ee335fab38247c1bfb0aa04fe341be2 2024-11-17T09:42:28,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/48199c90367a4ff389c056eecb8da5f2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/48199c90367a4ff389c056eecb8da5f2 2024-11-17T09:42:28,153 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/48199c90367a4ff389c056eecb8da5f2, entries=150, sequenceid=225, filesize=30.4 K 2024-11-17T09:42:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/5d71b623fc1349438090e0ef34ee2cb3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5d71b623fc1349438090e0ef34ee2cb3 2024-11-17T09:42:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,158 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5d71b623fc1349438090e0ef34ee2cb3, entries=150, sequenceid=225, filesize=11.9 K 2024-11-17T09:42:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/2ee335fab38247c1bfb0aa04fe341be2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/2ee335fab38247c1bfb0aa04fe341be2 2024-11-17T09:42:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,192 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/2ee335fab38247c1bfb0aa04fe341be2, entries=150, sequenceid=225, filesize=11.9 K 2024-11-17T09:42:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,193 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for d732b91b920f21f520735b3babf0e4b7 in 284ms, sequenceid=225, compaction requested=true 2024-11-17T09:42:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-17T09:42:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-17T09:42:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T09:42:28,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 441 msec 2024-11-17T09:42:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 448 msec 2024-11-17T09:42:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T09:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,380 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,381 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-17T09:42:28,383 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T09:42:28,384 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:28,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T09:42:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T09:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,540 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T09:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179417fa4f3a694cd99518622586c3f45e_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836548492/Put/seqid=0 2024-11-17T09:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742257_1433 (size=12304) 2024-11-17T09:42:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,559 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179417fa4f3a694cd99518622586c3f45e_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179417fa4f3a694cd99518622586c3f45e_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8a7658ed1c6843ba8b4738205c64b20f, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8a7658ed1c6843ba8b4738205c64b20f is 175, key is test_row_0/A:col10/1731836548492/Put/seqid=0 2024-11-17T09:42:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742258_1434 (size=31101) 2024-11-17T09:42:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:28,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T09:42:28,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836548680, max timestamp=1731836548757Num uniques 859; 2024-11-17T09:42:28,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836548680, max timestamp=1731836548757Num uniques 859; 2024-11-17T09:42:28,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836548680, max timestamp=1731836548757Num uniques 859; 2024-11-17T09:42:28,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:42:28,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:42:28,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:42:28,969 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8a7658ed1c6843ba8b4738205c64b20f 2024-11-17T09:42:28,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836548830, max timestamp=1731836548965Num uniques 1018; 2024-11-17T09:42:28,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836548830, max timestamp=1731836548965Num uniques 1018; 2024-11-17T09:42:28,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:42:28,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:42:28,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836548830, max timestamp=1731836548965Num uniques 1018; 2024-11-17T09:42:28,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:42:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/98ccc9b9d04d488192b57389645770a1 is 50, key is test_row_0/B:col10/1731836548492/Put/seqid=0 2024-11-17T09:42:28,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742259_1435 (size=9757) 2024-11-17T09:42:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T09:42:29,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836549072, max timestamp=1731836549145Num uniques 1027; 2024-11-17T09:42:29,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T09:42:29,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836549072, max timestamp=1731836549145Num uniques 1027; 2024-11-17T09:42:29,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T09:42:29,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731836549072, max timestamp=1731836549145Num uniques 1027; 2024-11-17T09:42:29,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T09:42:29,380 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/98ccc9b9d04d488192b57389645770a1 2024-11-17T09:42:29,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/95ebeef2f4d04de88bd2878dfde44130 is 50, key is test_row_0/C:col10/1731836548492/Put/seqid=0 2024-11-17T09:42:29,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742260_1436 (size=9757) 2024-11-17T09:42:29,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731836549284, max timestamp=1731836549416Num uniques 1036; 2024-11-17T09:42:29,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T09:42:29,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731836549284, max timestamp=1731836549416Num uniques 1036; 2024-11-17T09:42:29,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T09:42:29,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731836549284, max timestamp=1731836549416Num uniques 1036; 2024-11-17T09:42:29,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T09:42:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T09:42:29,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731836549599, max timestamp=1731836549645Num uniques 1045; 2024-11-17T09:42:29,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-17T09:42:29,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731836549599, max timestamp=1731836549645Num uniques 1045; 2024-11-17T09:42:29,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-17T09:42:29,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731836549599, max timestamp=1731836549645Num uniques 1045; 2024-11-17T09:42:29,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-17T09:42:29,791 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/95ebeef2f4d04de88bd2878dfde44130 2024-11-17T09:42:29,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/8a7658ed1c6843ba8b4738205c64b20f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8a7658ed1c6843ba8b4738205c64b20f 2024-11-17T09:42:29,798 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8a7658ed1c6843ba8b4738205c64b20f, entries=150, sequenceid=233, filesize=30.4 K 2024-11-17T09:42:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/98ccc9b9d04d488192b57389645770a1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/98ccc9b9d04d488192b57389645770a1 2024-11-17T09:42:29,802 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/98ccc9b9d04d488192b57389645770a1, entries=100, sequenceid=233, filesize=9.5 K 2024-11-17T09:42:29,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/95ebeef2f4d04de88bd2878dfde44130 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95ebeef2f4d04de88bd2878dfde44130 2024-11-17T09:42:29,805 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95ebeef2f4d04de88bd2878dfde44130, entries=100, sequenceid=233, filesize=9.5 K 2024-11-17T09:42:29,806 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=140.89 KB/144270 for d732b91b920f21f520735b3babf0e4b7 in 1267ms, sequenceid=233, compaction requested=true 2024-11-17T09:42:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-17T09:42:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-17T09:42:29,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-17T09:42:29,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4230 sec 2024-11-17T09:42:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:29,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=147.60 KB heapSize=422.16 KB 2024-11-17T09:42:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T09:42:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T09:42:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T09:42:29,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4280 sec 2024-11-17T09:42:29,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ed54388b3ef74a7f8297177200bec8c3_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836549773/Put/seqid=0 2024-11-17T09:42:29,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742261_1437 (size=14994) 2024-11-17T09:42:29,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836609868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836609990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44364 deadline: 1731836610100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:30,114 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:30,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1731836610110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:30,125 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44374 deadline: 1731836610120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:30,138 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44422 deadline: 1731836610129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:30,142 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in d732b91b920f21f520735b3babf0e4b7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:42:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,222 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:30,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836610210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,225 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ed54388b3ef74a7f8297177200bec8c3_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ed54388b3ef74a7f8297177200bec8c3_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:30,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/28bb54ded64b48dcbf263a5c91955b38, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:30,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/28bb54ded64b48dcbf263a5c91955b38 is 175, key is test_row_0/A:col10/1731836549773/Put/seqid=0 2024-11-17T09:42:30,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742262_1438 (size=39949) 2024-11-17T09:42:30,231 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/28bb54ded64b48dcbf263a5c91955b38 2024-11-17T09:42:30,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/1dcffd3613724731823f72fe90b82ec0 is 50, key is test_row_0/B:col10/1731836549773/Put/seqid=0 2024-11-17T09:42:30,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742263_1439 (size=12301) 2024-11-17T09:42:30,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T09:42:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T09:42:30,540 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:30,540 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T09:42:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T09:42:30,542 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:30,543 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:30,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:30,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836610529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:30,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:30,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/1dcffd3613724731823f72fe90b82ec0 2024-11-17T09:42:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T09:42:30,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/e93aa4cb72084ebca0e0170852be2489 is 50, key is test_row_0/C:col10/1731836549773/Put/seqid=0 2024-11-17T09:42:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742264_1440 (size=12301) 2024-11-17T09:42:30,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:30,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:30,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:30,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T09:42:31,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:31,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:31,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/e93aa4cb72084ebca0e0170852be2489 2024-11-17T09:42:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/28bb54ded64b48dcbf263a5c91955b38 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/28bb54ded64b48dcbf263a5c91955b38 2024-11-17T09:42:31,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/28bb54ded64b48dcbf263a5c91955b38, entries=200, sequenceid=340, filesize=39.0 K 2024-11-17T09:42:31,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/1dcffd3613724731823f72fe90b82ec0 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/1dcffd3613724731823f72fe90b82ec0 2024-11-17T09:42:31,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/1dcffd3613724731823f72fe90b82ec0, entries=150, sequenceid=340, filesize=12.0 K 2024-11-17T09:42:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/e93aa4cb72084ebca0e0170852be2489 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/e93aa4cb72084ebca0e0170852be2489 2024-11-17T09:42:31,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/e93aa4cb72084ebca0e0170852be2489, entries=150, sequenceid=340, filesize=12.0 K 2024-11-17T09:42:31,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~422.11 KB/432240, currentSize=33.54 KB/34350 for d732b91b920f21f520735b3babf0e4b7 in 1258ms, sequenceid=340, compaction requested=true 2024-11-17T09:42:31,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:31,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:31,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:31,069 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-17T09:42:31,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:31,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:31,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d732b91b920f21f520735b3babf0e4b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:31,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:31,069 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-17T09:42:31,076 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 83223 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-17T09:42:31,076 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/B is initiating minor compaction (all files) 2024-11-17T09:42:31,076 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 235629 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-17T09:42:31,077 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/B in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,077 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/A is initiating minor compaction (all files) 2024-11-17T09:42:31,077 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/A in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,077 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a5c4dcbd86d446128ca76884764ccacd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4ed47d8ed6248be9c246ae8428072db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/6c71a8a4bb1343eda2503996eb1a18ed, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/80d3e12c064147438b5f1c9fb6f51933, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5d71b623fc1349438090e0ef34ee2cb3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/98ccc9b9d04d488192b57389645770a1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/1dcffd3613724731823f72fe90b82ec0] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=81.3 K 2024-11-17T09:42:31,077 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e5d8aea1ec3e4599aeaeea37dcbaadd2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/23720bcc0f6c4507957162368e68d661, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8e2119135b26479788f5c05dfc62d199, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/22d44af61ff64a0c815d0fd811b4ad34, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/48199c90367a4ff389c056eecb8da5f2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8a7658ed1c6843ba8b4738205c64b20f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/28bb54ded64b48dcbf263a5c91955b38] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=230.1 K 2024-11-17T09:42:31,077 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,077 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e5d8aea1ec3e4599aeaeea37dcbaadd2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/23720bcc0f6c4507957162368e68d661, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8e2119135b26479788f5c05dfc62d199, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/22d44af61ff64a0c815d0fd811b4ad34, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/48199c90367a4ff389c056eecb8da5f2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8a7658ed1c6843ba8b4738205c64b20f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/28bb54ded64b48dcbf263a5c91955b38] 2024-11-17T09:42:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:31,077 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c4dcbd86d446128ca76884764ccacd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836541858 2024-11-17T09:42:31,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:42:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:31,078 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5d8aea1ec3e4599aeaeea37dcbaadd2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836541858 2024-11-17T09:42:31,078 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ed47d8ed6248be9c246ae8428072db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731836542646 2024-11-17T09:42:31,078 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c71a8a4bb1343eda2503996eb1a18ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731836543986 2024-11-17T09:42:31,078 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23720bcc0f6c4507957162368e68d661, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731836542646 2024-11-17T09:42:31,078 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e2119135b26479788f5c05dfc62d199, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731836543986 2024-11-17T09:42:31,079 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d3e12c064147438b5f1c9fb6f51933, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731836544704 2024-11-17T09:42:31,079 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d44af61ff64a0c815d0fd811b4ad34, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731836544704 2024-11-17T09:42:31,079 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d71b623fc1349438090e0ef34ee2cb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731836546004 2024-11-17T09:42:31,079 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48199c90367a4ff389c056eecb8da5f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731836546004 2024-11-17T09:42:31,079 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7658ed1c6843ba8b4738205c64b20f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836548380 2024-11-17T09:42:31,079 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ccc9b9d04d488192b57389645770a1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836548492 2024-11-17T09:42:31,080 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28bb54ded64b48dcbf263a5c91955b38, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731836549773 2024-11-17T09:42:31,080 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dcffd3613724731823f72fe90b82ec0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731836549773 2024-11-17T09:42:31,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b1dfebb47e0d4e70b98e8c98cf48f8de_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836551070/Put/seqid=0 2024-11-17T09:42:31,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742265_1441 (size=14994) 2024-11-17T09:42:31,103 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:31,104 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#B#compaction#370 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:31,105 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170fc0ffe4b5084056999a8204c40fddf2_d732b91b920f21f520735b3babf0e4b7 store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:31,105 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/044f4434e11343599f1da7311239cf9b is 50, key is test_row_0/B:col10/1731836549773/Put/seqid=0 2024-11-17T09:42:31,109 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170fc0ffe4b5084056999a8204c40fddf2_d732b91b920f21f520735b3babf0e4b7, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:31,109 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170fc0ffe4b5084056999a8204c40fddf2_d732b91b920f21f520735b3babf0e4b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:31,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742266_1442 (size=12949) 2024-11-17T09:42:31,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742267_1443 (size=4469) 2024-11-17T09:42:31,115 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/044f4434e11343599f1da7311239cf9b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/044f4434e11343599f1da7311239cf9b 2024-11-17T09:42:31,115 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#A#compaction#371 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:31,116 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/df45f5e1f73b4d2ea7ffb467759a5d5b is 175, key is test_row_0/A:col10/1731836549773/Put/seqid=0 2024-11-17T09:42:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742268_1444 (size=31903) 2024-11-17T09:42:31,120 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/B of d732b91b920f21f520735b3babf0e4b7 into 044f4434e11343599f1da7311239cf9b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:31,120 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:31,120 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/B, priority=9, startTime=1731836551069; duration=0sec 2024-11-17T09:42:31,120 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:31,120 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:B 2024-11-17T09:42:31,120 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-17T09:42:31,124 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 83223 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-17T09:42:31,125 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): d732b91b920f21f520735b3babf0e4b7/C is initiating minor compaction (all files) 2024-11-17T09:42:31,125 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d732b91b920f21f520735b3babf0e4b7/C in TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,125 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/098fc995a8094a13adfda9619aef9581, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95c5ab17c26d44039f25602616082554, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d8a8573d694c447b805a79f394248f6a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3007d3d33a6c43c38aac2b44913c0796, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/2ee335fab38247c1bfb0aa04fe341be2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95ebeef2f4d04de88bd2878dfde44130, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/e93aa4cb72084ebca0e0170852be2489] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp, totalSize=81.3 K 2024-11-17T09:42:31,125 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 098fc995a8094a13adfda9619aef9581, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731836541858 2024-11-17T09:42:31,125 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 95c5ab17c26d44039f25602616082554, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731836542646 2024-11-17T09:42:31,128 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a8573d694c447b805a79f394248f6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731836543986 2024-11-17T09:42:31,128 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3007d3d33a6c43c38aac2b44913c0796, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731836544704 2024-11-17T09:42:31,129 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ee335fab38247c1bfb0aa04fe341be2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731836546004 2024-11-17T09:42:31,129 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ebeef2f4d04de88bd2878dfde44130, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731836548492 2024-11-17T09:42:31,130 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e93aa4cb72084ebca0e0170852be2489, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731836549773 2024-11-17T09:42:31,130 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/df45f5e1f73b4d2ea7ffb467759a5d5b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/df45f5e1f73b4d2ea7ffb467759a5d5b 2024-11-17T09:42:31,134 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/A of d732b91b920f21f520735b3babf0e4b7 into df45f5e1f73b4d2ea7ffb467759a5d5b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:31,134 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:31,134 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/A, priority=9, startTime=1731836551069; duration=0sec 2024-11-17T09:42:31,134 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:31,134 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:A 2024-11-17T09:42:31,141 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d732b91b920f21f520735b3babf0e4b7#C#compaction#372 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:31,141 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/db3395236bd7496888427b8e7c7e7548 is 50, key is test_row_0/C:col10/1731836549773/Put/seqid=0 2024-11-17T09:42:31,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742269_1445 (size=12949) 2024-11-17T09:42:31,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T09:42:31,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836551197, max timestamp=1731836551246Num uniques 859; 2024-11-17T09:42:31,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:42:31,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:31,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:31,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:31,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836611310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:31,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:31,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836611419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:31,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:31,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:31,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:31,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b1dfebb47e0d4e70b98e8c98cf48f8de_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b1dfebb47e0d4e70b98e8c98cf48f8de_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:31,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/e56bb45404e54d8fbb40152f78aa6824, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:31,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/e56bb45404e54d8fbb40152f78aa6824 is 175, key is test_row_0/A:col10/1731836551070/Put/seqid=0 2024-11-17T09:42:31,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742270_1446 (size=39949) 2024-11-17T09:42:31,549 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/db3395236bd7496888427b8e7c7e7548 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/db3395236bd7496888427b8e7c7e7548 2024-11-17T09:42:31,553 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in d732b91b920f21f520735b3babf0e4b7/C of d732b91b920f21f520735b3babf0e4b7 into db3395236bd7496888427b8e7c7e7548(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:31,553 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:31,553 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., storeName=d732b91b920f21f520735b3babf0e4b7/C, priority=9, startTime=1731836551069; duration=0sec 2024-11-17T09:42:31,553 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:31,553 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d732b91b920f21f520735b3babf0e4b7:C 2024-11-17T09:42:31,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836611640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T09:42:31,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:31,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,844 INFO [Thread-1662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1662. 2024-11-17T09:42:31,844 DEBUG [Thread-1662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:31,844 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,844 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:31,846 INFO [Thread-1666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1666. 2024-11-17T09:42:31,846 DEBUG [Thread-1666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:31,846 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,847 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:31,849 INFO [Thread-1668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1668. 2024-11-17T09:42:31,849 DEBUG [Thread-1668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:31,849 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,849 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:31,858 INFO [Thread-1660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1660. 2024-11-17T09:42:31,858 DEBUG [Thread-1660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:31,858 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,858 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:31,858 INFO [Thread-1664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1664. 2024-11-17T09:42:31,858 DEBUG [Thread-1664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:31,858 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,858 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:31,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:31,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/e56bb45404e54d8fbb40152f78aa6824 2024-11-17T09:42:31,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/8bd3b07416344cc1a3d1547f25ab48c9 is 50, key is test_row_0/B:col10/1731836551070/Put/seqid=0 2024-11-17T09:42:31,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742271_1447 (size=12301) 2024-11-17T09:42:31,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:31,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:31,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:31,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836611959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:32,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:32,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:32,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:32,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:32,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/8bd3b07416344cc1a3d1547f25ab48c9 2024-11-17T09:42:32,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/3e1ae29a8ba24ce5988d34ef27cb4b80 is 50, key is test_row_0/C:col10/1731836551070/Put/seqid=0 2024-11-17T09:42:32,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742272_1448 (size=12301) 2024-11-17T09:42:32,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:32,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:32,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44398 deadline: 1731836612470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:32,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:42:32,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T09:42:32,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/3e1ae29a8ba24ce5988d34ef27cb4b80 2024-11-17T09:42:32,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/e56bb45404e54d8fbb40152f78aa6824 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e56bb45404e54d8fbb40152f78aa6824 2024-11-17T09:42:32,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e56bb45404e54d8fbb40152f78aa6824, entries=200, sequenceid=350, filesize=39.0 K 2024-11-17T09:42:32,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/8bd3b07416344cc1a3d1547f25ab48c9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/8bd3b07416344cc1a3d1547f25ab48c9 2024-11-17T09:42:32,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/8bd3b07416344cc1a3d1547f25ab48c9, entries=150, sequenceid=350, filesize=12.0 K 2024-11-17T09:42:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/3e1ae29a8ba24ce5988d34ef27cb4b80 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3e1ae29a8ba24ce5988d34ef27cb4b80 2024-11-17T09:42:32,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3e1ae29a8ba24ce5988d34ef27cb4b80, entries=150, sequenceid=350, filesize=12.0 K 2024-11-17T09:42:32,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for d732b91b920f21f520735b3babf0e4b7 in 1660ms, sequenceid=350, compaction requested=false 2024-11-17T09:42:32,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:32,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T09:42:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:32,836 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-11-17T09:42:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:42:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:42:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:32,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:42:32,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a9b6134968c44e5793139d72cacac9e7_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836551308/Put/seqid=0 2024-11-17T09:42:32,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742273_1449 (size=12454) 2024-11-17T09:42:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:33,248 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a9b6134968c44e5793139d72cacac9e7_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a9b6134968c44e5793139d72cacac9e7_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:33,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/15adf7efa49f47dd95881702df6943de, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:33,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/15adf7efa49f47dd95881702df6943de is 175, key is test_row_0/A:col10/1731836551308/Put/seqid=0 2024-11-17T09:42:33,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742274_1450 (size=31255) 2024-11-17T09:42:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:33,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. as already flushing 2024-11-17T09:42:33,481 INFO [Thread-1649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1649. 2024-11-17T09:42:33,481 DEBUG [Thread-1649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:33,481 DEBUG [Thread-1649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:33,481 DEBUG [Thread-1649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:33,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:33,653 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/15adf7efa49f47dd95881702df6943de 2024-11-17T09:42:33,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/3b14e3331ee24770a80864e5241d968e is 50, key is test_row_0/B:col10/1731836551308/Put/seqid=0 2024-11-17T09:42:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742275_1451 (size=12301) 2024-11-17T09:42:34,062 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/3b14e3331ee24770a80864e5241d968e 2024-11-17T09:42:34,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/20ca5ea54c174117bf9da21e70c08df8 is 50, key is test_row_0/C:col10/1731836551308/Put/seqid=0 2024-11-17T09:42:34,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742276_1452 (size=12301) 2024-11-17T09:42:34,402 WARN [regionserver/70e37c475432:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-17T09:42:34,403 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/c21810699d4b49e7adc682b398f41abc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4a82bba351ac4b1594204fe0d56fbf2b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3d25eba39eac411eb7e8db5eaed27892, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/b2fa22e943a141288cd844f8ceedee7d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/50ca44a75c954a93bad42fa7f463fae6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3c13a6a11886470992587b7f1f77e14b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/93d694dff5fb4443ad3455ac790e4e88, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/66a868f388384c78aec04727674a9d61, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/d388cc3c3f1f49fa8d793867fd2aa741, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4c745f315fdd4f58869d5e88d27186fd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/75371503b2a7475282ec54bfa072df60, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/cc1bbfc607ed4ab3842b937fad593a53, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e5d8aea1ec3e4599aeaeea37dcbaadd2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/23720bcc0f6c4507957162368e68d661, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8e2119135b26479788f5c05dfc62d199, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/22d44af61ff64a0c815d0fd811b4ad34, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/48199c90367a4ff389c056eecb8da5f2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8a7658ed1c6843ba8b4738205c64b20f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/28bb54ded64b48dcbf263a5c91955b38] to archive 2024-11-17T09:42:34,404 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:42:34,405 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/c21810699d4b49e7adc682b398f41abc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/c21810699d4b49e7adc682b398f41abc 2024-11-17T09:42:34,406 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4a82bba351ac4b1594204fe0d56fbf2b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4a82bba351ac4b1594204fe0d56fbf2b 2024-11-17T09:42:34,407 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3d25eba39eac411eb7e8db5eaed27892 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3d25eba39eac411eb7e8db5eaed27892 2024-11-17T09:42:34,408 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/b2fa22e943a141288cd844f8ceedee7d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/b2fa22e943a141288cd844f8ceedee7d 2024-11-17T09:42:34,409 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/50ca44a75c954a93bad42fa7f463fae6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/50ca44a75c954a93bad42fa7f463fae6 2024-11-17T09:42:34,410 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3c13a6a11886470992587b7f1f77e14b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/3c13a6a11886470992587b7f1f77e14b 2024-11-17T09:42:34,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/93d694dff5fb4443ad3455ac790e4e88 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/93d694dff5fb4443ad3455ac790e4e88 2024-11-17T09:42:34,411 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/66a868f388384c78aec04727674a9d61 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/66a868f388384c78aec04727674a9d61 2024-11-17T09:42:34,412 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/d388cc3c3f1f49fa8d793867fd2aa741 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/d388cc3c3f1f49fa8d793867fd2aa741 2024-11-17T09:42:34,413 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4c745f315fdd4f58869d5e88d27186fd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/4c745f315fdd4f58869d5e88d27186fd 2024-11-17T09:42:34,414 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/75371503b2a7475282ec54bfa072df60 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/75371503b2a7475282ec54bfa072df60 2024-11-17T09:42:34,415 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/cc1bbfc607ed4ab3842b937fad593a53 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/cc1bbfc607ed4ab3842b937fad593a53 2024-11-17T09:42:34,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e5d8aea1ec3e4599aeaeea37dcbaadd2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e5d8aea1ec3e4599aeaeea37dcbaadd2 2024-11-17T09:42:34,416 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/23720bcc0f6c4507957162368e68d661 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/23720bcc0f6c4507957162368e68d661 2024-11-17T09:42:34,417 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8e2119135b26479788f5c05dfc62d199 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8e2119135b26479788f5c05dfc62d199 2024-11-17T09:42:34,418 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/22d44af61ff64a0c815d0fd811b4ad34 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/22d44af61ff64a0c815d0fd811b4ad34 2024-11-17T09:42:34,419 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/48199c90367a4ff389c056eecb8da5f2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/48199c90367a4ff389c056eecb8da5f2 2024-11-17T09:42:34,420 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8a7658ed1c6843ba8b4738205c64b20f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/8a7658ed1c6843ba8b4738205c64b20f 2024-11-17T09:42:34,421 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/28bb54ded64b48dcbf263a5c91955b38 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/28bb54ded64b48dcbf263a5c91955b38 2024-11-17T09:42:34,423 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5e6740145fa04508a308b26e4fdca383, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b0c6c731d9ef4e28be9c4c5dc36edfb4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/ced8717298af4cd294491aa8d0c28d12, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/bc4e8c5e5f0b403291fcc0c949ae7668, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/0582e420d2a345a6990d2d6da0cfc8ea, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/15f3de55c3b3420897b551fe883bd13c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5a7488738140469bb0b18f5dce235eca, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c33c356a4886472ba73aa94c8d74e366, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a324a7734e5d4f3f8caa04db56818c01, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c8d388fed2e9401381beea7397603f47, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/672965e351284fb3b1436a8adf619939, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a5c4dcbd86d446128ca76884764ccacd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4027e81fe3f4bf58a16676553c1ce3e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4ed47d8ed6248be9c246ae8428072db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/6c71a8a4bb1343eda2503996eb1a18ed, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/80d3e12c064147438b5f1c9fb6f51933, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5d71b623fc1349438090e0ef34ee2cb3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/98ccc9b9d04d488192b57389645770a1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/1dcffd3613724731823f72fe90b82ec0] to archive 2024-11-17T09:42:34,424 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:42:34,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5e6740145fa04508a308b26e4fdca383 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5e6740145fa04508a308b26e4fdca383 2024-11-17T09:42:34,425 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b0c6c731d9ef4e28be9c4c5dc36edfb4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b0c6c731d9ef4e28be9c4c5dc36edfb4 2024-11-17T09:42:34,426 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/ced8717298af4cd294491aa8d0c28d12 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/ced8717298af4cd294491aa8d0c28d12 2024-11-17T09:42:34,427 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/bc4e8c5e5f0b403291fcc0c949ae7668 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/bc4e8c5e5f0b403291fcc0c949ae7668 2024-11-17T09:42:34,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/0582e420d2a345a6990d2d6da0cfc8ea to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/0582e420d2a345a6990d2d6da0cfc8ea 2024-11-17T09:42:34,428 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/15f3de55c3b3420897b551fe883bd13c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/15f3de55c3b3420897b551fe883bd13c 2024-11-17T09:42:34,429 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5a7488738140469bb0b18f5dce235eca to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5a7488738140469bb0b18f5dce235eca 2024-11-17T09:42:34,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c33c356a4886472ba73aa94c8d74e366 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c33c356a4886472ba73aa94c8d74e366 2024-11-17T09:42:34,430 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a324a7734e5d4f3f8caa04db56818c01 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a324a7734e5d4f3f8caa04db56818c01 2024-11-17T09:42:34,431 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c8d388fed2e9401381beea7397603f47 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/c8d388fed2e9401381beea7397603f47 2024-11-17T09:42:34,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/672965e351284fb3b1436a8adf619939 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/672965e351284fb3b1436a8adf619939 2024-11-17T09:42:34,432 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a5c4dcbd86d446128ca76884764ccacd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/a5c4dcbd86d446128ca76884764ccacd 2024-11-17T09:42:34,433 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4027e81fe3f4bf58a16676553c1ce3e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4027e81fe3f4bf58a16676553c1ce3e 2024-11-17T09:42:34,434 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4ed47d8ed6248be9c246ae8428072db to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/b4ed47d8ed6248be9c246ae8428072db 2024-11-17T09:42:34,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/6c71a8a4bb1343eda2503996eb1a18ed to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/6c71a8a4bb1343eda2503996eb1a18ed 2024-11-17T09:42:34,435 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/80d3e12c064147438b5f1c9fb6f51933 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/80d3e12c064147438b5f1c9fb6f51933 2024-11-17T09:42:34,436 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5d71b623fc1349438090e0ef34ee2cb3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/5d71b623fc1349438090e0ef34ee2cb3 2024-11-17T09:42:34,437 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/98ccc9b9d04d488192b57389645770a1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/98ccc9b9d04d488192b57389645770a1 2024-11-17T09:42:34,438 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/1dcffd3613724731823f72fe90b82ec0 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/1dcffd3613724731823f72fe90b82ec0 2024-11-17T09:42:34,440 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5ac02e7653364011ab8d5e33671ba96f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/21a29e3afcb140fcb3149b1e54b281a1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/f6d1eae22370475eb4fc06a30bc6cf6d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/29e37c07e3754ffe975405a70f1d59a5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/522bb5ab4cdd4f16b2a8f86c220baf8a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/bbccb5bdd2db4c11a55ece38dbc6e8e8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/b0152bf4c49a45988574c2c1868ccd86, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/dbbae89a4f9d411ba775a046942b2fa7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/ee4dca1872ae4f18b5db5224ff7f882c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/8437ef218e83446aa6784bcebb5c2869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d87e5be750a14b0392cb7e1bcc39f6e8, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/098fc995a8094a13adfda9619aef9581, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/a0201b61f7ce4cdb9c6f23c4374cc360, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95c5ab17c26d44039f25602616082554, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d8a8573d694c447b805a79f394248f6a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3007d3d33a6c43c38aac2b44913c0796, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/2ee335fab38247c1bfb0aa04fe341be2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95ebeef2f4d04de88bd2878dfde44130, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/e93aa4cb72084ebca0e0170852be2489] to archive 2024-11-17T09:42:34,441 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:42:34,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5ac02e7653364011ab8d5e33671ba96f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5ac02e7653364011ab8d5e33671ba96f 2024-11-17T09:42:34,442 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/21a29e3afcb140fcb3149b1e54b281a1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/21a29e3afcb140fcb3149b1e54b281a1 2024-11-17T09:42:34,443 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/f6d1eae22370475eb4fc06a30bc6cf6d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/f6d1eae22370475eb4fc06a30bc6cf6d 2024-11-17T09:42:34,444 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/29e37c07e3754ffe975405a70f1d59a5 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/29e37c07e3754ffe975405a70f1d59a5 2024-11-17T09:42:34,445 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/522bb5ab4cdd4f16b2a8f86c220baf8a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/522bb5ab4cdd4f16b2a8f86c220baf8a 2024-11-17T09:42:34,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/bbccb5bdd2db4c11a55ece38dbc6e8e8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/bbccb5bdd2db4c11a55ece38dbc6e8e8 2024-11-17T09:42:34,446 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/b0152bf4c49a45988574c2c1868ccd86 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/b0152bf4c49a45988574c2c1868ccd86 2024-11-17T09:42:34,447 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/dbbae89a4f9d411ba775a046942b2fa7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/dbbae89a4f9d411ba775a046942b2fa7 2024-11-17T09:42:34,448 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/ee4dca1872ae4f18b5db5224ff7f882c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/ee4dca1872ae4f18b5db5224ff7f882c 2024-11-17T09:42:34,449 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/8437ef218e83446aa6784bcebb5c2869 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/8437ef218e83446aa6784bcebb5c2869 2024-11-17T09:42:34,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d87e5be750a14b0392cb7e1bcc39f6e8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d87e5be750a14b0392cb7e1bcc39f6e8 2024-11-17T09:42:34,450 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/098fc995a8094a13adfda9619aef9581 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/098fc995a8094a13adfda9619aef9581 2024-11-17T09:42:34,451 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/a0201b61f7ce4cdb9c6f23c4374cc360 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/a0201b61f7ce4cdb9c6f23c4374cc360 2024-11-17T09:42:34,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95c5ab17c26d44039f25602616082554 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95c5ab17c26d44039f25602616082554 2024-11-17T09:42:34,453 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d8a8573d694c447b805a79f394248f6a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/d8a8573d694c447b805a79f394248f6a 2024-11-17T09:42:34,453 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3007d3d33a6c43c38aac2b44913c0796 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3007d3d33a6c43c38aac2b44913c0796 2024-11-17T09:42:34,454 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/2ee335fab38247c1bfb0aa04fe341be2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/2ee335fab38247c1bfb0aa04fe341be2 2024-11-17T09:42:34,455 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95ebeef2f4d04de88bd2878dfde44130 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/95ebeef2f4d04de88bd2878dfde44130 2024-11-17T09:42:34,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/e93aa4cb72084ebca0e0170852be2489 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/e93aa4cb72084ebca0e0170852be2489 2024-11-17T09:42:34,471 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/20ca5ea54c174117bf9da21e70c08df8 2024-11-17T09:42:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/15adf7efa49f47dd95881702df6943de as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/15adf7efa49f47dd95881702df6943de 2024-11-17T09:42:34,477 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/15adf7efa49f47dd95881702df6943de, entries=150, sequenceid=380, filesize=30.5 K 2024-11-17T09:42:34,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/3b14e3331ee24770a80864e5241d968e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/3b14e3331ee24770a80864e5241d968e 2024-11-17T09:42:34,480 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/3b14e3331ee24770a80864e5241d968e, entries=150, sequenceid=380, filesize=12.0 K 2024-11-17T09:42:34,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/20ca5ea54c174117bf9da21e70c08df8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/20ca5ea54c174117bf9da21e70c08df8 2024-11-17T09:42:34,483 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/20ca5ea54c174117bf9da21e70c08df8, entries=150, sequenceid=380, filesize=12.0 K 2024-11-17T09:42:34,484 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=6.71 KB/6870 for d732b91b920f21f520735b3babf0e4b7 in 1648ms, sequenceid=380, compaction requested=true 2024-11-17T09:42:34,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d732b91b920f21f520735b3babf0e4b7: 2024-11-17T09:42:34,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:34,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T09:42:34,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T09:42:34,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T09:42:34,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9420 sec 2024-11-17T09:42:34,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.9470 sec 2024-11-17T09:42:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T09:42:34,710 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:40,142 INFO [Thread-1655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1655. 2024-11-17T09:42:40,142 DEBUG [Thread-1655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:40,142 DEBUG [Thread-1655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,142 DEBUG [Thread-1655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:40,171 INFO [Thread-1653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1653. 2024-11-17T09:42:40,171 DEBUG [Thread-1653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:40,171 DEBUG [Thread-1653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,171 DEBUG [Thread-1653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,171 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:40,201 INFO [Thread-1651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1651. 2024-11-17T09:42:40,201 DEBUG [Thread-1651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:40,201 DEBUG [Thread-1651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,201 DEBUG [Thread-1651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:40,231 INFO [Thread-1657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1657. 2024-11-17T09:42:40,232 DEBUG [Thread-1657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:42:40,232 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,232 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 213 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 960 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2875 rows 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 968 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2899 rows 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 973 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2917 rows 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 969 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2900 rows 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 972 2024-11-17T09:42:40,232 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2911 rows 2024-11-17T09:42:40,232 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:42:40,232 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:42:40,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:40,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:42:40,233 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T09:42:40,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T09:42:40,236 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836560236"}]},"ts":"1731836560236"} 2024-11-17T09:42:40,237 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T09:42:40,237 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T09:42:40,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:42:40,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, UNASSIGN}] 2024-11-17T09:42:40,239 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, UNASSIGN 2024-11-17T09:42:40,240 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:40,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, UNASSIGN because future has completed 2024-11-17T09:42:40,241 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:42:40,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:42:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T09:42:40,393 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:40,393 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:42:40,393 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing d732b91b920f21f520735b3babf0e4b7, disabling compactions & flushes 2024-11-17T09:42:40,393 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:40,393 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:40,393 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. after waiting 0 ms 2024-11-17T09:42:40,393 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:40,393 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing d732b91b920f21f520735b3babf0e4b7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T09:42:40,394 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=A 2024-11-17T09:42:40,394 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:40,394 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=B 2024-11-17T09:42:40,394 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:40,394 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d732b91b920f21f520735b3babf0e4b7, store=C 2024-11-17T09:42:40,394 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:40,398 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117766e4d261909449abfa8302eac2b3d02_d732b91b920f21f520735b3babf0e4b7 is 50, key is test_row_0/A:col10/1731836560200/Put/seqid=0 2024-11-17T09:42:40,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742277_1453 (size=12454) 2024-11-17T09:42:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T09:42:40,802 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:40,806 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117766e4d261909449abfa8302eac2b3d02_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117766e4d261909449abfa8302eac2b3d02_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:40,806 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/dd2566cf66534d5bace317ee4b1d5afe, store: [table=TestAcidGuarantees family=A region=d732b91b920f21f520735b3babf0e4b7] 2024-11-17T09:42:40,807 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/dd2566cf66534d5bace317ee4b1d5afe is 175, key is test_row_0/A:col10/1731836560200/Put/seqid=0 2024-11-17T09:42:40,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742278_1454 (size=31255) 2024-11-17T09:42:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T09:42:41,211 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/dd2566cf66534d5bace317ee4b1d5afe 2024-11-17T09:42:41,217 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/dfa827120e984b548d44a9810559b359 is 50, key is test_row_0/B:col10/1731836560200/Put/seqid=0 2024-11-17T09:42:41,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742279_1455 (size=12301) 2024-11-17T09:42:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T09:42:41,620 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/dfa827120e984b548d44a9810559b359 2024-11-17T09:42:41,626 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/5b1c4bb273364c9fa49ac3ca58bb555c is 50, key is test_row_0/C:col10/1731836560200/Put/seqid=0 2024-11-17T09:42:41,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742280_1456 (size=12301) 2024-11-17T09:42:42,029 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/5b1c4bb273364c9fa49ac3ca58bb555c 2024-11-17T09:42:42,033 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/A/dd2566cf66534d5bace317ee4b1d5afe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/dd2566cf66534d5bace317ee4b1d5afe 2024-11-17T09:42:42,036 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/dd2566cf66534d5bace317ee4b1d5afe, entries=150, sequenceid=388, filesize=30.5 K 2024-11-17T09:42:42,036 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/B/dfa827120e984b548d44a9810559b359 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/dfa827120e984b548d44a9810559b359 2024-11-17T09:42:42,039 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/dfa827120e984b548d44a9810559b359, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T09:42:42,040 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/.tmp/C/5b1c4bb273364c9fa49ac3ca58bb555c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5b1c4bb273364c9fa49ac3ca58bb555c 2024-11-17T09:42:42,042 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5b1c4bb273364c9fa49ac3ca58bb555c, entries=150, sequenceid=388, filesize=12.0 K 2024-11-17T09:42:42,043 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d732b91b920f21f520735b3babf0e4b7 in 1650ms, sequenceid=388, compaction requested=true 2024-11-17T09:42:42,047 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/recovered.edits/391.seqid, newMaxSeqId=391, maxSeqId=4 2024-11-17T09:42:42,048 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. 2024-11-17T09:42:42,048 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for d732b91b920f21f520735b3babf0e4b7: Waiting for close lock at 1731836560393Running coprocessor pre-close hooks at 1731836560393Disabling compacts and flushes for region at 1731836560393Disabling writes for close at 1731836560393Obtaining lock to block concurrent updates at 1731836560393Preparing flush snapshotting stores in d732b91b920f21f520735b3babf0e4b7 at 1731836560393Finished memstore snapshotting TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731836560394 (+1 ms)Flushing stores of TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7. at 1731836560394Flushing d732b91b920f21f520735b3babf0e4b7/A: creating writer at 1731836560394Flushing mob file d732b91b920f21f520735b3babf0e4b7/A: appending metadata at 1731836560398 (+4 ms)Flushing mob file d732b91b920f21f520735b3babf0e4b7/A: closing flushed file at 1731836560398Flushing d732b91b920f21f520735b3babf0e4b7/A: appending metadata at 1731836560806 (+408 ms)Flushing d732b91b920f21f520735b3babf0e4b7/A: closing flushed file at 1731836560806Flushing d732b91b920f21f520735b3babf0e4b7/B: creating writer at 1731836561214 (+408 ms)Flushing d732b91b920f21f520735b3babf0e4b7/B: appending metadata at 1731836561216 (+2 ms)Flushing d732b91b920f21f520735b3babf0e4b7/B: closing flushed file at 1731836561216Flushing d732b91b920f21f520735b3babf0e4b7/C: creating writer at 1731836561624 (+408 ms)Flushing d732b91b920f21f520735b3babf0e4b7/C: appending metadata at 1731836561626 (+2 ms)Flushing d732b91b920f21f520735b3babf0e4b7/C: closing flushed file at 1731836561626Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54433c14: reopening flushed file at 1731836562032 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd105ad: reopening flushed file at 1731836562036 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39234488: reopening flushed file at 1731836562039 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d732b91b920f21f520735b3babf0e4b7 in 1650ms, sequenceid=388, compaction requested=true at 1731836562043 (+4 ms)Writing region close event to WAL at 1731836562045 (+2 ms)Running coprocessor post-close hooks at 1731836562048 (+3 ms)Closed at 1731836562048 2024-11-17T09:42:42,049 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=d732b91b920f21f520735b3babf0e4b7, regionState=CLOSED 2024-11-17T09:42:42,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:42:42,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T09:42:42,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure d732b91b920f21f520735b3babf0e4b7, server=70e37c475432,40671,1731836432285 in 1.8110 sec 2024-11-17T09:42:42,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-17T09:42:42,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d732b91b920f21f520735b3babf0e4b7, UNASSIGN in 1.8140 sec 2024-11-17T09:42:42,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T09:42:42,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8170 sec 2024-11-17T09:42:42,057 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836562057"}]},"ts":"1731836562057"} 2024-11-17T09:42:42,058 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T09:42:42,058 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T09:42:42,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8260 sec 2024-11-17T09:42:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T09:42:42,380 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:42,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T09:42:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:42,382 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T09:42:42,382 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:42,385 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,386 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/recovered.edits] 2024-11-17T09:42:42,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/15adf7efa49f47dd95881702df6943de to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/15adf7efa49f47dd95881702df6943de 2024-11-17T09:42:42,389 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/dd2566cf66534d5bace317ee4b1d5afe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/dd2566cf66534d5bace317ee4b1d5afe 2024-11-17T09:42:42,390 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/df45f5e1f73b4d2ea7ffb467759a5d5b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/df45f5e1f73b4d2ea7ffb467759a5d5b 2024-11-17T09:42:42,390 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e56bb45404e54d8fbb40152f78aa6824 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/A/e56bb45404e54d8fbb40152f78aa6824 2024-11-17T09:42:42,392 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/044f4434e11343599f1da7311239cf9b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/044f4434e11343599f1da7311239cf9b 2024-11-17T09:42:42,393 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/3b14e3331ee24770a80864e5241d968e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/3b14e3331ee24770a80864e5241d968e 2024-11-17T09:42:42,393 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/8bd3b07416344cc1a3d1547f25ab48c9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/8bd3b07416344cc1a3d1547f25ab48c9 2024-11-17T09:42:42,394 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/dfa827120e984b548d44a9810559b359 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/B/dfa827120e984b548d44a9810559b359 2024-11-17T09:42:42,396 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/20ca5ea54c174117bf9da21e70c08df8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/20ca5ea54c174117bf9da21e70c08df8 2024-11-17T09:42:42,396 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3e1ae29a8ba24ce5988d34ef27cb4b80 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/3e1ae29a8ba24ce5988d34ef27cb4b80 2024-11-17T09:42:42,397 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5b1c4bb273364c9fa49ac3ca58bb555c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/5b1c4bb273364c9fa49ac3ca58bb555c 2024-11-17T09:42:42,398 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/db3395236bd7496888427b8e7c7e7548 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/C/db3395236bd7496888427b8e7c7e7548 2024-11-17T09:42:42,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/recovered.edits/391.seqid to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7/recovered.edits/391.seqid 2024-11-17T09:42:42,400 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,400 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T09:42:42,400 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T09:42:42,401 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T09:42:42,403 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117098d390ea8304797b7a715ef5dcd6200_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117098d390ea8304797b7a715ef5dcd6200_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111729ac117aaf7044b9add37fd29d363442_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111729ac117aaf7044b9add37fd29d363442_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,405 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172fda10f8ed2f4606bb3b889f6c200a93_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172fda10f8ed2f4606bb3b889f6c200a93_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175988cbb9881b45f3bcc9736df0425a69_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175988cbb9881b45f3bcc9736df0425a69_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117660c697acb2746cda1d888f7b4c2cfb6_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117660c697acb2746cda1d888f7b4c2cfb6_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117766e4d261909449abfa8302eac2b3d02_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117766e4d261909449abfa8302eac2b3d02_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,408 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179417fa4f3a694cd99518622586c3f45e_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179417fa4f3a694cd99518622586c3f45e_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,409 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a37736aa123246e6b37fefefd9beac34_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a37736aa123246e6b37fefefd9beac34_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a9b6134968c44e5793139d72cacac9e7_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a9b6134968c44e5793139d72cacac9e7_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b1dfebb47e0d4e70b98e8c98cf48f8de_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b1dfebb47e0d4e70b98e8c98cf48f8de_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c58badd44d4e4afb91c6cf3758ebbdff_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c58badd44d4e4afb91c6cf3758ebbdff_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d5305619c8f546d19a97cc7f790a3ccb_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d5305619c8f546d19a97cc7f790a3ccb_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,413 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d92b76b3b46a4b4cafb22108a16d8b72_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d92b76b3b46a4b4cafb22108a16d8b72_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,413 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dc0b1ace366441819ea118cdb3e5727b_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dc0b1ace366441819ea118cdb3e5727b_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e8bf3e49ea6b45dd8cc74a161722962b_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e8bf3e49ea6b45dd8cc74a161722962b_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,415 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eacd82fb936848a590795ce11b0a5148_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eacd82fb936848a590795ce11b0a5148_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,415 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ed54388b3ef74a7f8297177200bec8c3_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ed54388b3ef74a7f8297177200bec8c3_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,416 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3c1d3023acc40a09ee17421070f149a_d732b91b920f21f520735b3babf0e4b7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f3c1d3023acc40a09ee17421070f149a_d732b91b920f21f520735b3babf0e4b7 2024-11-17T09:42:42,417 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T09:42:42,418 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:42,420 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T09:42:42,422 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T09:42:42,423 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:42,423 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T09:42:42,423 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731836562423"}]},"ts":"9223372036854775807"} 2024-11-17T09:42:42,424 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T09:42:42,424 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d732b91b920f21f520735b3babf0e4b7, NAME => 'TestAcidGuarantees,,1731836528721.d732b91b920f21f520735b3babf0e4b7.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T09:42:42,424 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T09:42:42,425 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731836562424"}]},"ts":"9223372036854775807"} 2024-11-17T09:42:42,426 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T09:42:42,426 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:42,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-17T09:42:42,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T09:42:42,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T09:42:42,490 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:42,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 244) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 367), ProcessCount=11 (was 11), AvailableMemoryMB=2349 (was 2948) 2024-11-17T09:42:42,509 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=361, ProcessCount=11, AvailableMemoryMB=2349 2024-11-17T09:42:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:42:42,511 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:42:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T09:42:42,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:42:42,513 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:42,513 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-17T09:42:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T09:42:42,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:42:42,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742281_1457 (size=1061) 2024-11-17T09:42:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T09:42:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T09:42:42,920 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:42:42,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742282_1458 (size=53) 2024-11-17T09:42:43,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T09:42:43,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:42:43,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fe8ebf75400bab2f775cc2a034eb525d, disabling compactions & flushes 2024-11-17T09:42:43,325 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. after waiting 0 ms 2024-11-17T09:42:43,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,325 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,325 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe8ebf75400bab2f775cc2a034eb525d: Waiting for close lock at 1731836563325Disabling compacts and flushes for region at 1731836563325Disabling writes for close at 1731836563325Writing region close event to WAL at 1731836563325Closed at 1731836563325 2024-11-17T09:42:43,326 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:42:43,327 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731836563327"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836563327"}]},"ts":"1731836563327"} 2024-11-17T09:42:43,328 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:42:43,329 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:42:43,329 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836563329"}]},"ts":"1731836563329"} 2024-11-17T09:42:43,330 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T09:42:43,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, ASSIGN}] 2024-11-17T09:42:43,331 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, ASSIGN 2024-11-17T09:42:43,331 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, ASSIGN; state=OFFLINE, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=false 2024-11-17T09:42:43,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=fe8ebf75400bab2f775cc2a034eb525d, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:43,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, ASSIGN because future has completed 2024-11-17T09:42:43,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:42:43,638 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,638 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:42:43,639 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,639 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:42:43,639 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,639 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,640 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,641 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:43,641 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8ebf75400bab2f775cc2a034eb525d columnFamilyName A 2024-11-17T09:42:43,641 DEBUG [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:43,642 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.HStore(327): Store=fe8ebf75400bab2f775cc2a034eb525d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:43,642 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,643 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:43,643 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8ebf75400bab2f775cc2a034eb525d columnFamilyName B 2024-11-17T09:42:43,643 DEBUG [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:43,643 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.HStore(327): Store=fe8ebf75400bab2f775cc2a034eb525d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:43,643 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,644 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:42:43,644 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe8ebf75400bab2f775cc2a034eb525d columnFamilyName C 2024-11-17T09:42:43,644 DEBUG [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:42:43,644 INFO [StoreOpener-fe8ebf75400bab2f775cc2a034eb525d-1 {}] regionserver.HStore(327): Store=fe8ebf75400bab2f775cc2a034eb525d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:42:43,645 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,645 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,645 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,646 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,646 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,646 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,647 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:42:43,648 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T09:42:43,650 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:42:43,650 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened fe8ebf75400bab2f775cc2a034eb525d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72377163, jitterRate=0.07850377261638641}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:42:43,650 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:43,651 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for fe8ebf75400bab2f775cc2a034eb525d: Running coprocessor pre-open hook at 1731836563639Writing region info on filesystem at 1731836563639Initializing all the Stores at 1731836563640 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836563640Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836563640Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836563640Cleaning up temporary data from old regions at 1731836563646 (+6 ms)Running coprocessor post-open hooks at 1731836563650 (+4 ms)Region opened successfully at 1731836563651 (+1 ms) 2024-11-17T09:42:43,652 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., pid=118, masterSystemTime=1731836563635 2024-11-17T09:42:43,653 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,653 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:43,654 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=fe8ebf75400bab2f775cc2a034eb525d, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:42:43,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:42:43,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T09:42:43,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 in 173 msec 2024-11-17T09:42:43,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-17T09:42:43,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, ASSIGN in 327 msec 2024-11-17T09:42:43,659 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:42:43,659 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836563659"}]},"ts":"1731836563659"} 2024-11-17T09:42:43,660 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T09:42:43,661 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:42:43,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-17T09:42:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-17T09:42:44,660 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:44,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c44e9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78879d63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,664 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,665 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:42:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,44533,1731836431485 2024-11-17T09:42:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d9277cf 2024-11-17T09:42:44,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:42:44,667 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:42:44,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,669 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0ba03c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575ed406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,672 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a15b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a5c0b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,674 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@df28036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,675 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3cb5e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,676 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573f912e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51425745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a2aeb02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73bbc8e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,682 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6b8f53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,683 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1c2887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,684 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c956b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc4f912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6f4632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,687 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154826e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5909eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ee40ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:42:44,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:42:44,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:42:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:42:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:42:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c8c41ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:42:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:42:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:42:44,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:42:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710a9b4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:42:44,693 DEBUG [Thread-2058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,693 DEBUG [Thread-2073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,693 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,693 DEBUG [Thread-2060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,694 DEBUG [Thread-2062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,694 DEBUG [Thread-2056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,694 DEBUG [Thread-2075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,695 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,695 DEBUG [Thread-2064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:44,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,697 DEBUG [Thread-2071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:42:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:42:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:42:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:42:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-17T09:42:44,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2] 2024-11-17T09:42:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T09:42:44,705 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:44,706 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:44,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:44,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:42:44,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:44,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:44,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:44,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:44,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:44,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:44,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/827c44f0fd4e4314aa3c44d8d5488398 is 50, key is test_row_0/A:col10/1731836564709/Put/seqid=0 2024-11-17T09:42:44,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742283_1459 (size=12001) 2024-11-17T09:42:44,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/827c44f0fd4e4314aa3c44d8d5488398 2024-11-17T09:42:44,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836624752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836624753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836624754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836624760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/505d4e76fc3b4bfcbf65221c617b90c5 is 50, key is test_row_0/B:col10/1731836564709/Put/seqid=0 2024-11-17T09:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742284_1460 (size=12001) 2024-11-17T09:42:44,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836624761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T09:42:44,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T09:42:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:44,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:44,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:44,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:44,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:44,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836624870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836624879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836624879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836624879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836624880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:44,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:44,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T09:42:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T09:42:45,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836625079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836625100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836625100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836625100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836625100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T09:42:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/505d4e76fc3b4bfcbf65221c617b90c5 2024-11-17T09:42:45,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/d141fcb365e845c7bc330c6814d03d98 is 50, key is test_row_0/C:col10/1731836564709/Put/seqid=0 2024-11-17T09:42:45,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742285_1461 (size=12001) 2024-11-17T09:42:45,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T09:42:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T09:42:45,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836625399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836625420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836625420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836625420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836625420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T09:42:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/d141fcb365e845c7bc330c6814d03d98 2024-11-17T09:42:45,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/827c44f0fd4e4314aa3c44d8d5488398 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/827c44f0fd4e4314aa3c44d8d5488398 2024-11-17T09:42:45,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/827c44f0fd4e4314aa3c44d8d5488398, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T09:42:45,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/505d4e76fc3b4bfcbf65221c617b90c5 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/505d4e76fc3b4bfcbf65221c617b90c5 2024-11-17T09:42:45,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/505d4e76fc3b4bfcbf65221c617b90c5, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T09:42:45,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/d141fcb365e845c7bc330c6814d03d98 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d141fcb365e845c7bc330c6814d03d98 2024-11-17T09:42:45,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T09:42:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:45,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d141fcb365e845c7bc330c6814d03d98, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T09:42:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fe8ebf75400bab2f775cc2a034eb525d in 916ms, sequenceid=13, compaction requested=false 2024-11-17T09:42:45,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:45,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-17T09:42:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:45,777 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:42:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:45,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/07192f504874430584af3c56410ef9b4 is 50, key is test_row_0/A:col10/1731836564758/Put/seqid=0 2024-11-17T09:42:45,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742286_1462 (size=12001) 2024-11-17T09:42:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T09:42:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:45,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:45,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836625944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836625946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836625947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836625948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836625949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836626059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836626059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836626070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836626070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836626070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,185 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/07192f504874430584af3c56410ef9b4 2024-11-17T09:42:46,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/0f0e3bc501854019b41bb6aa7f6a4194 is 50, key is test_row_0/B:col10/1731836564758/Put/seqid=0 2024-11-17T09:42:46,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742287_1463 (size=12001) 2024-11-17T09:42:46,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836626269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836626280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836626280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836626280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836626280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836626589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836626590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836626590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,602 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/0f0e3bc501854019b41bb6aa7f6a4194 2024-11-17T09:42:46,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836626600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836626600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:46,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/ab4eaab7339140ad90dd1b6c206fbc95 is 50, key is test_row_0/C:col10/1731836564758/Put/seqid=0 2024-11-17T09:42:46,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742288_1464 (size=12001) 2024-11-17T09:42:46,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T09:42:47,013 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/ab4eaab7339140ad90dd1b6c206fbc95 2024-11-17T09:42:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/07192f504874430584af3c56410ef9b4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/07192f504874430584af3c56410ef9b4 2024-11-17T09:42:47,020 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/07192f504874430584af3c56410ef9b4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-17T09:42:47,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/0f0e3bc501854019b41bb6aa7f6a4194 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/0f0e3bc501854019b41bb6aa7f6a4194 2024-11-17T09:42:47,024 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/0f0e3bc501854019b41bb6aa7f6a4194, entries=150, sequenceid=33, filesize=11.7 K 2024-11-17T09:42:47,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/ab4eaab7339140ad90dd1b6c206fbc95 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab4eaab7339140ad90dd1b6c206fbc95 2024-11-17T09:42:47,028 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab4eaab7339140ad90dd1b6c206fbc95, entries=150, sequenceid=33, filesize=11.7 K 2024-11-17T09:42:47,028 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fe8ebf75400bab2f775cc2a034eb525d in 1251ms, sequenceid=33, compaction requested=false 2024-11-17T09:42:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-17T09:42:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-17T09:42:47,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-17T09:42:47,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-17T09:42:47,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.3350 sec 2024-11-17T09:42:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:47,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T09:42:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:47,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/fd36bd364ef1464585765497f22cd11f is 50, key is test_row_0/A:col10/1731836567102/Put/seqid=0 2024-11-17T09:42:47,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742289_1465 (size=12001) 2024-11-17T09:42:47,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836627136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836627137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836627139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836627144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836627146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836627260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836627260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836627260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836627260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836627269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836627480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836627480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836627480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836627480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836627480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/fd36bd364ef1464585765497f22cd11f 2024-11-17T09:42:47,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/ee7b423b202a4f8692dc4920a9348159 is 50, key is test_row_0/B:col10/1731836567102/Put/seqid=0 2024-11-17T09:42:47,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742290_1466 (size=12001) 2024-11-17T09:42:47,565 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:42:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836627800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836627800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836627800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836627800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836627800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:47,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/ee7b423b202a4f8692dc4920a9348159 2024-11-17T09:42:47,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/78a441422d7640dc8a63311ee1e508ac is 50, key is test_row_0/C:col10/1731836567102/Put/seqid=0 2024-11-17T09:42:47,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742291_1467 (size=12001) 2024-11-17T09:42:48,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836628310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:48,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836628320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:48,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836628320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:48,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836628320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:48,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836628320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:48,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:48,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/78a441422d7640dc8a63311ee1e508ac 2024-11-17T09:42:48,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/fd36bd364ef1464585765497f22cd11f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fd36bd364ef1464585765497f22cd11f 2024-11-17T09:42:48,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fd36bd364ef1464585765497f22cd11f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T09:42:48,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/ee7b423b202a4f8692dc4920a9348159 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/ee7b423b202a4f8692dc4920a9348159 2024-11-17T09:42:48,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/ee7b423b202a4f8692dc4920a9348159, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T09:42:48,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/78a441422d7640dc8a63311ee1e508ac as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/78a441422d7640dc8a63311ee1e508ac 2024-11-17T09:42:48,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/78a441422d7640dc8a63311ee1e508ac, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T09:42:48,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fe8ebf75400bab2f775cc2a034eb525d in 1267ms, sequenceid=46, compaction requested=true 2024-11-17T09:42:48,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:48,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:48,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:48,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:48,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:48,372 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:48,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:48,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:42:48,372 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:48,373 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:48,373 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/A is initiating minor compaction (all files) 2024-11-17T09:42:48,373 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:48,373 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/A in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:48,373 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/B is initiating minor compaction (all files) 2024-11-17T09:42:48,373 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/827c44f0fd4e4314aa3c44d8d5488398, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/07192f504874430584af3c56410ef9b4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fd36bd364ef1464585765497f22cd11f] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.2 K 2024-11-17T09:42:48,373 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/B in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:48,373 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/505d4e76fc3b4bfcbf65221c617b90c5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/0f0e3bc501854019b41bb6aa7f6a4194, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/ee7b423b202a4f8692dc4920a9348159] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.2 K 2024-11-17T09:42:48,373 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 827c44f0fd4e4314aa3c44d8d5488398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731836564705 2024-11-17T09:42:48,373 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 505d4e76fc3b4bfcbf65221c617b90c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731836564705 2024-11-17T09:42:48,374 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07192f504874430584af3c56410ef9b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731836564737 2024-11-17T09:42:48,374 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f0e3bc501854019b41bb6aa7f6a4194, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731836564737 2024-11-17T09:42:48,374 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd36bd364ef1464585765497f22cd11f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836565946 2024-11-17T09:42:48,374 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ee7b423b202a4f8692dc4920a9348159, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836565946 2024-11-17T09:42:48,403 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:48,403 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/b6063a3c8539453198cc0734c5f7c42c is 50, key is test_row_0/A:col10/1731836567102/Put/seqid=0 2024-11-17T09:42:48,411 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:48,411 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/c4e0b59bb6f040bcab3cc73ee6e37c87 is 50, key is test_row_0/B:col10/1731836567102/Put/seqid=0 2024-11-17T09:42:48,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742292_1468 (size=12104) 2024-11-17T09:42:48,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742293_1469 (size=12104) 2024-11-17T09:42:48,458 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/b6063a3c8539453198cc0734c5f7c42c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/b6063a3c8539453198cc0734c5f7c42c 2024-11-17T09:42:48,460 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/c4e0b59bb6f040bcab3cc73ee6e37c87 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/c4e0b59bb6f040bcab3cc73ee6e37c87 2024-11-17T09:42:48,466 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/B of fe8ebf75400bab2f775cc2a034eb525d into c4e0b59bb6f040bcab3cc73ee6e37c87(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:48,466 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:48,466 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/B, priority=13, startTime=1731836568372; duration=0sec 2024-11-17T09:42:48,466 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:48,466 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:B 2024-11-17T09:42:48,466 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:48,467 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/A of fe8ebf75400bab2f775cc2a034eb525d into b6063a3c8539453198cc0734c5f7c42c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:48,467 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:48,467 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/A, priority=13, startTime=1731836568371; duration=0sec 2024-11-17T09:42:48,467 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:48,467 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:A 2024-11-17T09:42:48,467 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:48,467 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/C is initiating minor compaction (all files) 2024-11-17T09:42:48,467 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/C in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:48,467 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d141fcb365e845c7bc330c6814d03d98, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab4eaab7339140ad90dd1b6c206fbc95, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/78a441422d7640dc8a63311ee1e508ac] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.2 K 2024-11-17T09:42:48,468 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d141fcb365e845c7bc330c6814d03d98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731836564705 2024-11-17T09:42:48,468 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4eaab7339140ad90dd1b6c206fbc95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731836564737 2024-11-17T09:42:48,468 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a441422d7640dc8a63311ee1e508ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836565946 2024-11-17T09:42:48,487 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:48,487 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/394b687f2b2142978fa175c8c983a463 is 50, key is test_row_0/C:col10/1731836567102/Put/seqid=0 2024-11-17T09:42:48,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742294_1470 (size=12104) 2024-11-17T09:42:48,529 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/394b687f2b2142978fa175c8c983a463 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/394b687f2b2142978fa175c8c983a463 2024-11-17T09:42:48,538 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/C of fe8ebf75400bab2f775cc2a034eb525d into 394b687f2b2142978fa175c8c983a463(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:48,538 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:48,538 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/C, priority=13, startTime=1731836568372; duration=0sec 2024-11-17T09:42:48,539 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:48,539 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:C 2024-11-17T09:42:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T09:42:48,860 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:48,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-17T09:42:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T09:42:48,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:48,863 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:48,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T09:42:49,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-17T09:42:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:49,016 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T09:42:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:49,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/735e1e6ea0ad430f930b8a12e9748616 is 50, key is test_row_0/A:col10/1731836567143/Put/seqid=0 2024-11-17T09:42:49,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742295_1471 (size=12001) 2024-11-17T09:42:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T09:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:49,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836629352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836629353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836629357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836629358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836629359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,425 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/735e1e6ea0ad430f930b8a12e9748616 2024-11-17T09:42:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5a2a71c6f61645709b8bc8ce821f38a6 is 50, key is test_row_0/B:col10/1731836567143/Put/seqid=0 2024-11-17T09:42:49,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742296_1472 (size=12001) 2024-11-17T09:42:49,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836629470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836629470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836629470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836629480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836629480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T09:42:49,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836629679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836629680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836629680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836629690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836629690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:49,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:49,838 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5a2a71c6f61645709b8bc8ce821f38a6 2024-11-17T09:42:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8648aa1919e3461894980d04c2192c88 is 50, key is test_row_0/C:col10/1731836567143/Put/seqid=0 2024-11-17T09:42:49,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742297_1473 (size=12001) 2024-11-17T09:42:49,849 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8648aa1919e3461894980d04c2192c88 2024-11-17T09:42:49,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/735e1e6ea0ad430f930b8a12e9748616 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/735e1e6ea0ad430f930b8a12e9748616 2024-11-17T09:42:49,858 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/735e1e6ea0ad430f930b8a12e9748616, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T09:42:49,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5a2a71c6f61645709b8bc8ce821f38a6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5a2a71c6f61645709b8bc8ce821f38a6 2024-11-17T09:42:49,862 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5a2a71c6f61645709b8bc8ce821f38a6, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T09:42:49,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8648aa1919e3461894980d04c2192c88 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8648aa1919e3461894980d04c2192c88 2024-11-17T09:42:49,866 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8648aa1919e3461894980d04c2192c88, entries=150, sequenceid=68, filesize=11.7 K 2024-11-17T09:42:49,866 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fe8ebf75400bab2f775cc2a034eb525d in 851ms, sequenceid=68, compaction requested=false 2024-11-17T09:42:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-17T09:42:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-17T09:42:49,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T09:42:49,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-17T09:42:49,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0100 sec 2024-11-17T09:42:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-17T09:42:50,000 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:50,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-17T09:42:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T09:42:50,002 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:50,003 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:50,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:50,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T09:42:50,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:50,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:50,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:50,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:50,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:50,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:50,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/0878b8f3098145f9bd5097c229c64930 is 50, key is test_row_0/A:col10/1731836569358/Put/seqid=0 2024-11-17T09:42:50,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742298_1474 (size=16681) 2024-11-17T09:42:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836630058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836630058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836630059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836630059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836630061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T09:42:50,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:50,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836630179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836630179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836630180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836630180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836630180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:50,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:50,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T09:42:50,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836630400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836630400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836630400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836630400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836630400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/0878b8f3098145f9bd5097c229c64930 2024-11-17T09:42:50,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/1128c31d4c054107a92ff1198cc3d3ec is 50, key is test_row_0/B:col10/1731836569358/Put/seqid=0 2024-11-17T09:42:50,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742299_1475 (size=12001) 2024-11-17T09:42:50,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:50,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:50,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:50,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:50,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T09:42:50,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836630720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836630720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836630720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836630720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836630720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:50,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:50,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/1128c31d4c054107a92ff1198cc3d3ec 2024-11-17T09:42:50,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/968a924377e349d19d682c20d36481ac is 50, key is test_row_0/C:col10/1731836569358/Put/seqid=0 2024-11-17T09:42:50,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742300_1476 (size=12001) 2024-11-17T09:42:50,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:50,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:51,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:51,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:51,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:51,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T09:42:51,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836631229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:51,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836631239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:51,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836631240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:51,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836631240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836631240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:51,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:51,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/968a924377e349d19d682c20d36481ac 2024-11-17T09:42:51,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/0878b8f3098145f9bd5097c229c64930 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0878b8f3098145f9bd5097c229c64930 2024-11-17T09:42:51,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0878b8f3098145f9bd5097c229c64930, entries=250, sequenceid=84, filesize=16.3 K 2024-11-17T09:42:51,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/1128c31d4c054107a92ff1198cc3d3ec as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1128c31d4c054107a92ff1198cc3d3ec 2024-11-17T09:42:51,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1128c31d4c054107a92ff1198cc3d3ec, entries=150, sequenceid=84, filesize=11.7 K 2024-11-17T09:42:51,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/968a924377e349d19d682c20d36481ac as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/968a924377e349d19d682c20d36481ac 2024-11-17T09:42:51,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/968a924377e349d19d682c20d36481ac, entries=150, sequenceid=84, filesize=11.7 K 2024-11-17T09:42:51,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe8ebf75400bab2f775cc2a034eb525d in 1258ms, sequenceid=84, compaction requested=true 2024-11-17T09:42:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:51,284 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:51,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:51,284 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:51,285 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:51,285 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:51,285 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/B is initiating minor compaction (all files) 2024-11-17T09:42:51,285 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/A is initiating minor compaction (all files) 2024-11-17T09:42:51,285 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/B in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,285 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/A in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,285 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/c4e0b59bb6f040bcab3cc73ee6e37c87, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5a2a71c6f61645709b8bc8ce821f38a6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1128c31d4c054107a92ff1198cc3d3ec] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.3 K 2024-11-17T09:42:51,285 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/b6063a3c8539453198cc0734c5f7c42c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/735e1e6ea0ad430f930b8a12e9748616, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0878b8f3098145f9bd5097c229c64930] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=39.8 K 2024-11-17T09:42:51,286 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e0b59bb6f040bcab3cc73ee6e37c87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836565946 2024-11-17T09:42:51,286 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6063a3c8539453198cc0734c5f7c42c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836565946 2024-11-17T09:42:51,286 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a2a71c6f61645709b8bc8ce821f38a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731836567124 2024-11-17T09:42:51,286 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 735e1e6ea0ad430f930b8a12e9748616, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731836567124 2024-11-17T09:42:51,287 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1128c31d4c054107a92ff1198cc3d3ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836569358 2024-11-17T09:42:51,287 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0878b8f3098145f9bd5097c229c64930, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836569355 2024-11-17T09:42:51,294 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:51,295 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/a7fdcc8f2abb4f87bafd4fdafdd907c1 is 50, key is test_row_0/B:col10/1731836569358/Put/seqid=0 2024-11-17T09:42:51,298 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#A#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:51,298 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/91fe9283bacd4517a19c03073ee19420 is 50, key is test_row_0/A:col10/1731836569358/Put/seqid=0 2024-11-17T09:42:51,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742301_1477 (size=12207) 2024-11-17T09:42:51,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742302_1478 (size=12207) 2024-11-17T09:42:51,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T09:42:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,378 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/5389ef44df7b4bcfa12bd3fe7a51f7ce is 50, key is test_row_0/A:col10/1731836570059/Put/seqid=0 2024-11-17T09:42:51,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742303_1479 (size=12001) 2024-11-17T09:42:51,711 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/a7fdcc8f2abb4f87bafd4fdafdd907c1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a7fdcc8f2abb4f87bafd4fdafdd907c1 2024-11-17T09:42:51,715 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/B of fe8ebf75400bab2f775cc2a034eb525d into a7fdcc8f2abb4f87bafd4fdafdd907c1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:51,715 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:51,715 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/B, priority=13, startTime=1731836571284; duration=0sec 2024-11-17T09:42:51,715 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:51,716 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:B 2024-11-17T09:42:51,716 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:51,716 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/91fe9283bacd4517a19c03073ee19420 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/91fe9283bacd4517a19c03073ee19420 2024-11-17T09:42:51,717 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:51,717 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/C is initiating minor compaction (all files) 2024-11-17T09:42:51,717 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/C in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:51,717 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/394b687f2b2142978fa175c8c983a463, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8648aa1919e3461894980d04c2192c88, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/968a924377e349d19d682c20d36481ac] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.3 K 2024-11-17T09:42:51,718 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 394b687f2b2142978fa175c8c983a463, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836565946 2024-11-17T09:42:51,718 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8648aa1919e3461894980d04c2192c88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731836567124 2024-11-17T09:42:51,719 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 968a924377e349d19d682c20d36481ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836569358 2024-11-17T09:42:51,722 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/A of fe8ebf75400bab2f775cc2a034eb525d into 91fe9283bacd4517a19c03073ee19420(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:51,722 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:51,722 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/A, priority=13, startTime=1731836571284; duration=0sec 2024-11-17T09:42:51,722 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:51,722 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:A 2024-11-17T09:42:51,726 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:51,727 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/d6a2cfed42934a19b5922bfc0452e87d is 50, key is test_row_0/C:col10/1731836569358/Put/seqid=0 2024-11-17T09:42:51,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742304_1480 (size=12207) 2024-11-17T09:42:51,787 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/5389ef44df7b4bcfa12bd3fe7a51f7ce 2024-11-17T09:42:51,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/479b31cc63464a4db9af78b85ef69ef1 is 50, key is test_row_0/B:col10/1731836570059/Put/seqid=0 2024-11-17T09:42:51,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742305_1481 (size=12001) 2024-11-17T09:42:52,136 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/d6a2cfed42934a19b5922bfc0452e87d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d6a2cfed42934a19b5922bfc0452e87d 2024-11-17T09:42:52,140 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/C of fe8ebf75400bab2f775cc2a034eb525d into d6a2cfed42934a19b5922bfc0452e87d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:52,140 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:52,140 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/C, priority=13, startTime=1731836571284; duration=0sec 2024-11-17T09:42:52,140 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:52,140 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:C 2024-11-17T09:42:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T09:42:52,198 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/479b31cc63464a4db9af78b85ef69ef1 2024-11-17T09:42:52,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8f12bf55698e490c9cb08e1dccb7417c is 50, key is test_row_0/C:col10/1731836570059/Put/seqid=0 2024-11-17T09:42:52,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742306_1482 (size=12001) 2024-11-17T09:42:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:52,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836632270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836632272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836632280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836632280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836632281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836632390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836632390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836632390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836632399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836632400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,609 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8f12bf55698e490c9cb08e1dccb7417c 2024-11-17T09:42:52,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/5389ef44df7b4bcfa12bd3fe7a51f7ce as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5389ef44df7b4bcfa12bd3fe7a51f7ce 2024-11-17T09:42:52,616 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5389ef44df7b4bcfa12bd3fe7a51f7ce, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T09:42:52,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/479b31cc63464a4db9af78b85ef69ef1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/479b31cc63464a4db9af78b85ef69ef1 2024-11-17T09:42:52,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836632610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836632610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836632610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836632610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836632610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,621 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/479b31cc63464a4db9af78b85ef69ef1, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T09:42:52,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8f12bf55698e490c9cb08e1dccb7417c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8f12bf55698e490c9cb08e1dccb7417c 2024-11-17T09:42:52,624 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8f12bf55698e490c9cb08e1dccb7417c, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T09:42:52,625 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe8ebf75400bab2f775cc2a034eb525d in 1248ms, sequenceid=100, compaction requested=false 2024-11-17T09:42:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-17T09:42:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-17T09:42:52,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T09:42:52,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-11-17T09:42:52,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6280 sec 2024-11-17T09:42:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:52,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:42:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:52,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:52,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/47f6a606b45c4820a8e85b3ff77e35cd is 50, key is test_row_0/A:col10/1731836572934/Put/seqid=0 2024-11-17T09:42:52,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742307_1483 (size=12001) 2024-11-17T09:42:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836632981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836632982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836632987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:52,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836632991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836632997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836633099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836633100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836633100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836633110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836633110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836633320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836633320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836633320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836633330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836633330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/47f6a606b45c4820a8e85b3ff77e35cd 2024-11-17T09:42:53,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5eefe3a5556a4b1faa4a69b014ba482b is 50, key is test_row_0/B:col10/1731836572934/Put/seqid=0 2024-11-17T09:42:53,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742308_1484 (size=12001) 2024-11-17T09:42:53,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836633639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836633640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836633640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836633650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836633650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:53,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5eefe3a5556a4b1faa4a69b014ba482b 2024-11-17T09:42:53,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/08bbfae918fa4c128ef60423383f237b is 50, key is test_row_0/C:col10/1731836572934/Put/seqid=0 2024-11-17T09:42:53,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742309_1485 (size=12001) 2024-11-17T09:42:53,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/08bbfae918fa4c128ef60423383f237b 2024-11-17T09:42:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/47f6a606b45c4820a8e85b3ff77e35cd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/47f6a606b45c4820a8e85b3ff77e35cd 2024-11-17T09:42:53,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/47f6a606b45c4820a8e85b3ff77e35cd, entries=150, sequenceid=120, filesize=11.7 K 2024-11-17T09:42:53,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5eefe3a5556a4b1faa4a69b014ba482b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eefe3a5556a4b1faa4a69b014ba482b 2024-11-17T09:42:53,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eefe3a5556a4b1faa4a69b014ba482b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-17T09:42:53,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/08bbfae918fa4c128ef60423383f237b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/08bbfae918fa4c128ef60423383f237b 2024-11-17T09:42:53,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/08bbfae918fa4c128ef60423383f237b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-17T09:42:53,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fe8ebf75400bab2f775cc2a034eb525d in 877ms, sequenceid=120, compaction requested=true 2024-11-17T09:42:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:53,813 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:53,813 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:53,813 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:53,813 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/A is initiating minor compaction (all files) 2024-11-17T09:42:53,814 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/A in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:53,814 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/91fe9283bacd4517a19c03073ee19420, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5389ef44df7b4bcfa12bd3fe7a51f7ce, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/47f6a606b45c4820a8e85b3ff77e35cd] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.4 K 2024-11-17T09:42:53,814 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:53,814 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91fe9283bacd4517a19c03073ee19420, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836569358 2024-11-17T09:42:53,814 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/B is initiating minor compaction (all files) 2024-11-17T09:42:53,814 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/B in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:53,814 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a7fdcc8f2abb4f87bafd4fdafdd907c1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/479b31cc63464a4db9af78b85ef69ef1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eefe3a5556a4b1faa4a69b014ba482b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.4 K 2024-11-17T09:42:53,814 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5389ef44df7b4bcfa12bd3fe7a51f7ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731836570056 2024-11-17T09:42:53,814 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a7fdcc8f2abb4f87bafd4fdafdd907c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836569358 2024-11-17T09:42:53,815 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47f6a606b45c4820a8e85b3ff77e35cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731836572266 2024-11-17T09:42:53,815 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 479b31cc63464a4db9af78b85ef69ef1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731836570056 2024-11-17T09:42:53,815 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eefe3a5556a4b1faa4a69b014ba482b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731836572266 2024-11-17T09:42:53,824 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:53,824 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/bc37d17a0a16466dbb9bf91bb8b7e4a5 is 50, key is test_row_0/A:col10/1731836572934/Put/seqid=0 2024-11-17T09:42:53,825 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:53,825 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/a0825c76b6eb49cf8d2f2e5bae29bd1e is 50, key is test_row_0/B:col10/1731836572934/Put/seqid=0 2024-11-17T09:42:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742311_1487 (size=12309) 2024-11-17T09:42:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742310_1486 (size=12309) 2024-11-17T09:42:53,839 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/a0825c76b6eb49cf8d2f2e5bae29bd1e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a0825c76b6eb49cf8d2f2e5bae29bd1e 2024-11-17T09:42:53,843 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/bc37d17a0a16466dbb9bf91bb8b7e4a5 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc37d17a0a16466dbb9bf91bb8b7e4a5 2024-11-17T09:42:53,844 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/B of fe8ebf75400bab2f775cc2a034eb525d into a0825c76b6eb49cf8d2f2e5bae29bd1e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:53,844 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:53,844 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/B, priority=13, startTime=1731836573813; duration=0sec 2024-11-17T09:42:53,844 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:53,844 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:B 2024-11-17T09:42:53,844 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:42:53,846 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:42:53,846 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/C is initiating minor compaction (all files) 2024-11-17T09:42:53,847 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/C in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:53,847 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d6a2cfed42934a19b5922bfc0452e87d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8f12bf55698e490c9cb08e1dccb7417c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/08bbfae918fa4c128ef60423383f237b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=35.4 K 2024-11-17T09:42:53,847 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a2cfed42934a19b5922bfc0452e87d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836569358 2024-11-17T09:42:53,848 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f12bf55698e490c9cb08e1dccb7417c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731836570056 2024-11-17T09:42:53,848 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 08bbfae918fa4c128ef60423383f237b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731836572266 2024-11-17T09:42:53,849 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/A of fe8ebf75400bab2f775cc2a034eb525d into bc37d17a0a16466dbb9bf91bb8b7e4a5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:53,849 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:53,849 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/A, priority=13, startTime=1731836573813; duration=0sec 2024-11-17T09:42:53,849 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:53,849 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:A 2024-11-17T09:42:53,853 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:53,853 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/c4585969687d4e4ebc687989a3c1dfe1 is 50, key is test_row_0/C:col10/1731836572934/Put/seqid=0 2024-11-17T09:42:53,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742312_1488 (size=12309) 2024-11-17T09:42:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:54,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:54,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/fb89c6260e844c758fe6cfc4da8c0d77 is 50, key is test_row_0/A:col10/1731836572989/Put/seqid=0 2024-11-17T09:42:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T09:42:54,170 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:54,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-17T09:42:54,172 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T09:42:54,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742313_1489 (size=14491) 2024-11-17T09:42:54,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/fb89c6260e844c758fe6cfc4da8c0d77 2024-11-17T09:42:54,173 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:54,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:54,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6c1235ec7e1449a89e1d8bf63eb5bf06 is 50, key is test_row_0/B:col10/1731836572989/Put/seqid=0 2024-11-17T09:42:54,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742314_1490 (size=12151) 2024-11-17T09:42:54,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836634182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836634190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836634193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836634194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836634196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,265 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/c4585969687d4e4ebc687989a3c1dfe1 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c4585969687d4e4ebc687989a3c1dfe1 2024-11-17T09:42:54,269 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/C of fe8ebf75400bab2f775cc2a034eb525d into c4585969687d4e4ebc687989a3c1dfe1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:54,269 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:54,269 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/C, priority=13, startTime=1731836573813; duration=0sec 2024-11-17T09:42:54,269 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:54,269 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:C 2024-11-17T09:42:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T09:42:54,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836634300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836634310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836634310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836634310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T09:42:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836634319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T09:42:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T09:42:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836634510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836634519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836634520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836634520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836634540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6c1235ec7e1449a89e1d8bf63eb5bf06 2024-11-17T09:42:54,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/eee8f4b345914db4a282d35fcc882870 is 50, key is test_row_0/C:col10/1731836572989/Put/seqid=0 2024-11-17T09:42:54,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742315_1491 (size=12151) 2024-11-17T09:42:54,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T09:42:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T09:42:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T09:42:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836634829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836634829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836634829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836634840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836634850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:54,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T09:42:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:54,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:54,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/eee8f4b345914db4a282d35fcc882870 2024-11-17T09:42:55,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/fb89c6260e844c758fe6cfc4da8c0d77 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fb89c6260e844c758fe6cfc4da8c0d77 2024-11-17T09:42:55,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fb89c6260e844c758fe6cfc4da8c0d77, entries=200, sequenceid=138, filesize=14.2 K 2024-11-17T09:42:55,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6c1235ec7e1449a89e1d8bf63eb5bf06 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c1235ec7e1449a89e1d8bf63eb5bf06 2024-11-17T09:42:55,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c1235ec7e1449a89e1d8bf63eb5bf06, entries=150, sequenceid=138, filesize=11.9 K 2024-11-17T09:42:55,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/eee8f4b345914db4a282d35fcc882870 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/eee8f4b345914db4a282d35fcc882870 2024-11-17T09:42:55,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/eee8f4b345914db4a282d35fcc882870, entries=150, sequenceid=138, filesize=11.9 K 2024-11-17T09:42:55,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe8ebf75400bab2f775cc2a034eb525d in 847ms, sequenceid=138, compaction requested=false 2024-11-17T09:42:55,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:55,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T09:42:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:55,091 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T09:42:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/f5c1a2dc927d4577bca599410e605c2e is 50, key is test_row_0/A:col10/1731836574193/Put/seqid=0 2024-11-17T09:42:55,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742316_1492 (size=12151) 2024-11-17T09:42:55,100 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/f5c1a2dc927d4577bca599410e605c2e 2024-11-17T09:42:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/61e0549f3d7f4d96ba6f397c7daf359b is 50, key is test_row_0/B:col10/1731836574193/Put/seqid=0 2024-11-17T09:42:55,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742317_1493 (size=12151) 2024-11-17T09:42:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T09:42:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:55,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:55,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836635372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836635372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836635373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836635377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836635378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836635489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836635490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836635490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836635490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836635500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,510 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/61e0549f3d7f4d96ba6f397c7daf359b 2024-11-17T09:42:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/f7718f83cc8744bea13da1ffabbe9f25 is 50, key is test_row_0/C:col10/1731836574193/Put/seqid=0 2024-11-17T09:42:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742318_1494 (size=12151) 2024-11-17T09:42:55,527 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/f7718f83cc8744bea13da1ffabbe9f25 2024-11-17T09:42:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/f5c1a2dc927d4577bca599410e605c2e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/f5c1a2dc927d4577bca599410e605c2e 2024-11-17T09:42:55,535 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/f5c1a2dc927d4577bca599410e605c2e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T09:42:55,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/61e0549f3d7f4d96ba6f397c7daf359b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/61e0549f3d7f4d96ba6f397c7daf359b 2024-11-17T09:42:55,541 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/61e0549f3d7f4d96ba6f397c7daf359b, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T09:42:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/f7718f83cc8744bea13da1ffabbe9f25 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/f7718f83cc8744bea13da1ffabbe9f25 2024-11-17T09:42:55,545 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/f7718f83cc8744bea13da1ffabbe9f25, entries=150, sequenceid=155, filesize=11.9 K 2024-11-17T09:42:55,546 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe8ebf75400bab2f775cc2a034eb525d in 455ms, sequenceid=155, compaction requested=true 2024-11-17T09:42:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:55,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-17T09:42:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-17T09:42:55,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T09:42:55,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3740 sec 2024-11-17T09:42:55,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3790 sec 2024-11-17T09:42:55,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:55,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T09:42:55,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:55,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:55,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:55,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:55,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:55,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:55,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/e1f13af01ed94c85adfc186be4faa671 is 50, key is test_row_0/A:col10/1731836575371/Put/seqid=0 2024-11-17T09:42:55,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742319_1495 (size=16931) 2024-11-17T09:42:55,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836635736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836635737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836635738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836635741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836635752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836635859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836635859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836635859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836635870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836635870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836636080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836636080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836636080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836636090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836636090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/e1f13af01ed94c85adfc186be4faa671 2024-11-17T09:42:56,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6c749be6e6c44ab689c5c86d3fe5dc41 is 50, key is test_row_0/B:col10/1731836575371/Put/seqid=0 2024-11-17T09:42:56,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742320_1496 (size=12151) 2024-11-17T09:42:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T09:42:56,320 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:56,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T09:42:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T09:42:56,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:56,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:56,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836636399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836636400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836636400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836636400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836636400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T09:42:56,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T09:42:56,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6c749be6e6c44ab689c5c86d3fe5dc41 2024-11-17T09:42:56,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8d0bcb0f83a44b6093731f2e27940b7b is 50, key is test_row_0/C:col10/1731836575371/Put/seqid=0 2024-11-17T09:42:56,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742321_1497 (size=12151) 2024-11-17T09:42:56,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T09:42:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T09:42:56,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T09:42:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836636920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836636920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T09:42:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:42:56,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836636929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836636929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836636930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T09:42:56,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8d0bcb0f83a44b6093731f2e27940b7b 2024-11-17T09:42:56,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/e1f13af01ed94c85adfc186be4faa671 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/e1f13af01ed94c85adfc186be4faa671 2024-11-17T09:42:56,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/e1f13af01ed94c85adfc186be4faa671, entries=250, sequenceid=173, filesize=16.5 K 2024-11-17T09:42:56,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6c749be6e6c44ab689c5c86d3fe5dc41 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c749be6e6c44ab689c5c86d3fe5dc41 2024-11-17T09:42:56,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c749be6e6c44ab689c5c86d3fe5dc41, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T09:42:56,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8d0bcb0f83a44b6093731f2e27940b7b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8d0bcb0f83a44b6093731f2e27940b7b 2024-11-17T09:42:56,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8d0bcb0f83a44b6093731f2e27940b7b, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T09:42:56,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fe8ebf75400bab2f775cc2a034eb525d in 1248ms, sequenceid=173, compaction requested=true 2024-11-17T09:42:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:42:56,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:56,968 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:42:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:42:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:42:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:56,969 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55882 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/A is initiating minor compaction (all files) 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/B is initiating minor compaction (all files) 2024-11-17T09:42:56,970 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/B in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,970 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/A in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:56,970 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a0825c76b6eb49cf8d2f2e5bae29bd1e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c1235ec7e1449a89e1d8bf63eb5bf06, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/61e0549f3d7f4d96ba6f397c7daf359b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c749be6e6c44ab689c5c86d3fe5dc41] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=47.6 K 2024-11-17T09:42:56,970 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc37d17a0a16466dbb9bf91bb8b7e4a5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fb89c6260e844c758fe6cfc4da8c0d77, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/f5c1a2dc927d4577bca599410e605c2e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/e1f13af01ed94c85adfc186be4faa671] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=54.6 K 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a0825c76b6eb49cf8d2f2e5bae29bd1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731836572266 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc37d17a0a16466dbb9bf91bb8b7e4a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731836572266 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c1235ec7e1449a89e1d8bf63eb5bf06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731836572989 2024-11-17T09:42:56,970 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb89c6260e844c758fe6cfc4da8c0d77, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731836572975 2024-11-17T09:42:56,971 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e0549f3d7f4d96ba6f397c7daf359b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731836574191 2024-11-17T09:42:56,971 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c1a2dc927d4577bca599410e605c2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731836574191 2024-11-17T09:42:56,971 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c749be6e6c44ab689c5c86d3fe5dc41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731836575371 2024-11-17T09:42:56,971 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f13af01ed94c85adfc186be4faa671, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731836575357 2024-11-17T09:42:56,980 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#B#compaction#420 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:56,980 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/84ef08dfc0464a9a96e056f80f239b79 is 50, key is test_row_0/B:col10/1731836575371/Put/seqid=0 2024-11-17T09:42:56,982 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#A#compaction#421 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:56,982 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/0488bb7960db47309af94cce3d8ee7ba is 50, key is test_row_0/A:col10/1731836575371/Put/seqid=0 2024-11-17T09:42:56,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742323_1499 (size=12595) 2024-11-17T09:42:56,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742322_1498 (size=12595) 2024-11-17T09:42:57,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T09:42:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:57,088 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T09:42:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/a84b1a9fe3c845429cab19ef0633c3de is 50, key is test_row_0/A:col10/1731836575740/Put/seqid=0 2024-11-17T09:42:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742324_1500 (size=12151) 2024-11-17T09:42:57,396 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/0488bb7960db47309af94cce3d8ee7ba as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0488bb7960db47309af94cce3d8ee7ba 2024-11-17T09:42:57,400 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/84ef08dfc0464a9a96e056f80f239b79 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/84ef08dfc0464a9a96e056f80f239b79 2024-11-17T09:42:57,401 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/A of fe8ebf75400bab2f775cc2a034eb525d into 0488bb7960db47309af94cce3d8ee7ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:57,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:57,401 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/A, priority=12, startTime=1731836576968; duration=0sec 2024-11-17T09:42:57,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:42:57,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:A 2024-11-17T09:42:57,401 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:42:57,402 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:42:57,402 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/C is initiating minor compaction (all files) 2024-11-17T09:42:57,402 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/C in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:57,402 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c4585969687d4e4ebc687989a3c1dfe1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/eee8f4b345914db4a282d35fcc882870, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/f7718f83cc8744bea13da1ffabbe9f25, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8d0bcb0f83a44b6093731f2e27940b7b] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=47.6 K 2024-11-17T09:42:57,403 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4585969687d4e4ebc687989a3c1dfe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731836572266 2024-11-17T09:42:57,403 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/B of fe8ebf75400bab2f775cc2a034eb525d into 84ef08dfc0464a9a96e056f80f239b79(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:57,403 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:57,403 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting eee8f4b345914db4a282d35fcc882870, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731836572989 2024-11-17T09:42:57,403 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/B, priority=12, startTime=1731836576968; duration=0sec 2024-11-17T09:42:57,404 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:57,404 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:B 2024-11-17T09:42:57,404 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7718f83cc8744bea13da1ffabbe9f25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731836574191 2024-11-17T09:42:57,404 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d0bcb0f83a44b6093731f2e27940b7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731836575371 2024-11-17T09:42:57,411 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:42:57,412 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/1ab65d4c6fcf41b3812c961a12787ae6 is 50, key is test_row_0/C:col10/1731836575371/Put/seqid=0 2024-11-17T09:42:57,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742325_1501 (size=12595) 2024-11-17T09:42:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T09:42:57,499 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/a84b1a9fe3c845429cab19ef0633c3de 2024-11-17T09:42:57,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/76e803a5fa894442a0c1bea44d1561db is 50, key is test_row_0/B:col10/1731836575740/Put/seqid=0 2024-11-17T09:42:57,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742326_1502 (size=12151) 2024-11-17T09:42:57,820 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/1ab65d4c6fcf41b3812c961a12787ae6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/1ab65d4c6fcf41b3812c961a12787ae6 2024-11-17T09:42:57,824 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/C of fe8ebf75400bab2f775cc2a034eb525d into 1ab65d4c6fcf41b3812c961a12787ae6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:42:57,824 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:57,824 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/C, priority=12, startTime=1731836576969; duration=0sec 2024-11-17T09:42:57,824 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:42:57,824 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:C 2024-11-17T09:42:57,910 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/76e803a5fa894442a0c1bea44d1561db 2024-11-17T09:42:57,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/ab120298924a43dcbf293d9d1c850ca2 is 50, key is test_row_0/C:col10/1731836575740/Put/seqid=0 2024-11-17T09:42:57,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742327_1503 (size=12151) 2024-11-17T09:42:57,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:57,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:57,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:57,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836637974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836637973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836637976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836637980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:57,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:57,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836637989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:57,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836638100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836638100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836638100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836638100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836638101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836638310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836638310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836638310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836638311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836638311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,334 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/ab120298924a43dcbf293d9d1c850ca2 2024-11-17T09:42:58,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/a84b1a9fe3c845429cab19ef0633c3de as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/a84b1a9fe3c845429cab19ef0633c3de 2024-11-17T09:42:58,342 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/a84b1a9fe3c845429cab19ef0633c3de, entries=150, sequenceid=187, filesize=11.9 K 2024-11-17T09:42:58,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/76e803a5fa894442a0c1bea44d1561db as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/76e803a5fa894442a0c1bea44d1561db 2024-11-17T09:42:58,346 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/76e803a5fa894442a0c1bea44d1561db, entries=150, sequenceid=187, filesize=11.9 K 2024-11-17T09:42:58,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/ab120298924a43dcbf293d9d1c850ca2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab120298924a43dcbf293d9d1c850ca2 2024-11-17T09:42:58,351 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab120298924a43dcbf293d9d1c850ca2, entries=150, sequenceid=187, filesize=11.9 K 2024-11-17T09:42:58,352 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fe8ebf75400bab2f775cc2a034eb525d in 1264ms, sequenceid=187, compaction requested=false 2024-11-17T09:42:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:58,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T09:42:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T09:42:58,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T09:42:58,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0300 sec 2024-11-17T09:42:58,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0350 sec 2024-11-17T09:42:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T09:42:58,470 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:42:58,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:42:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T09:42:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T09:42:58,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:42:58,474 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:42:58,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:42:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T09:42:58,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T09:42:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:58,626 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T09:42:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:42:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:42:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:42:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:42:58,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/9ec08a0ae1e2487bb8682bdda49910e9 is 50, key is test_row_0/A:col10/1731836577969/Put/seqid=0 2024-11-17T09:42:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:42:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:42:58,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742328_1504 (size=12151) 2024-11-17T09:42:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836638648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836638649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836638650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836638652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836638655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836638759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836638770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836638770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836638770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T09:42:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836638780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836638980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836638980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836638980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836638980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836638999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,035 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/9ec08a0ae1e2487bb8682bdda49910e9 2024-11-17T09:42:59,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5eed0a057438447d8d600295241fa84a is 50, key is test_row_0/B:col10/1731836577969/Put/seqid=0 2024-11-17T09:42:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742329_1505 (size=12151) 2024-11-17T09:42:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T09:42:59,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836639290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836639290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836639290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836639290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836639319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,448 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5eed0a057438447d8d600295241fa84a 2024-11-17T09:42:59,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/18c2e09d0a684a11bdfbd1aa3dd594a6 is 50, key is test_row_0/C:col10/1731836577969/Put/seqid=0 2024-11-17T09:42:59,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742330_1506 (size=12151) 2024-11-17T09:42:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T09:42:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836639800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836639809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836639809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836639809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:42:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836639830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:42:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:42:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:42:59,859 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/18c2e09d0a684a11bdfbd1aa3dd594a6 2024-11-17T09:42:59,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/9ec08a0ae1e2487bb8682bdda49910e9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9ec08a0ae1e2487bb8682bdda49910e9 2024-11-17T09:42:59,867 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9ec08a0ae1e2487bb8682bdda49910e9, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T09:42:59,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5eed0a057438447d8d600295241fa84a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eed0a057438447d8d600295241fa84a 2024-11-17T09:42:59,872 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eed0a057438447d8d600295241fa84a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T09:42:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/18c2e09d0a684a11bdfbd1aa3dd594a6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/18c2e09d0a684a11bdfbd1aa3dd594a6 2024-11-17T09:42:59,876 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/18c2e09d0a684a11bdfbd1aa3dd594a6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T09:42:59,877 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fe8ebf75400bab2f775cc2a034eb525d in 1251ms, sequenceid=208, compaction requested=true 2024-11-17T09:42:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:42:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:42:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T09:42:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T09:42:59,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T09:42:59,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-17T09:42:59,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4100 sec 2024-11-17T09:43:00,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T09:43:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T09:43:00,620 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:00,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T09:43:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T09:43:00,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:00,623 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:00,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T09:43:00,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T09:43:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:00,776 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T09:43:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:43:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:43:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:43:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:00,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/775df0bf12e844bd9d0e3323a9b2c0ef is 50, key is test_row_0/A:col10/1731836578648/Put/seqid=0 2024-11-17T09:43:00,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742331_1507 (size=12151) 2024-11-17T09:43:00,783 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/775df0bf12e844bd9d0e3323a9b2c0ef 2024-11-17T09:43:00,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/db5d730c671d4455adc8412bfb629c4d is 50, key is test_row_0/B:col10/1731836578648/Put/seqid=0 2024-11-17T09:43:00,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742332_1508 (size=12151) 2024-11-17T09:43:00,799 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/db5d730c671d4455adc8412bfb629c4d 2024-11-17T09:43:00,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b87d612f3a404499871bc6e75e6912c2 is 50, key is test_row_0/C:col10/1731836578648/Put/seqid=0 2024-11-17T09:43:00,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742333_1509 (size=12151) 2024-11-17T09:43:00,814 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b87d612f3a404499871bc6e75e6912c2 2024-11-17T09:43:00,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/775df0bf12e844bd9d0e3323a9b2c0ef as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/775df0bf12e844bd9d0e3323a9b2c0ef 2024-11-17T09:43:00,822 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/775df0bf12e844bd9d0e3323a9b2c0ef, entries=150, sequenceid=222, filesize=11.9 K 2024-11-17T09:43:00,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/db5d730c671d4455adc8412bfb629c4d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/db5d730c671d4455adc8412bfb629c4d 2024-11-17T09:43:00,826 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/db5d730c671d4455adc8412bfb629c4d, entries=150, sequenceid=222, filesize=11.9 K 2024-11-17T09:43:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b87d612f3a404499871bc6e75e6912c2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b87d612f3a404499871bc6e75e6912c2 2024-11-17T09:43:00,831 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b87d612f3a404499871bc6e75e6912c2, entries=150, sequenceid=222, filesize=11.9 K 2024-11-17T09:43:00,831 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for fe8ebf75400bab2f775cc2a034eb525d in 55ms, sequenceid=222, compaction requested=true 2024-11-17T09:43:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T09:43:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T09:43:00,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T09:43:00,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec 2024-11-17T09:43:00,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 216 msec 2024-11-17T09:43:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:00,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:43:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:43:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:43:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:43:00,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:00,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/731a3adfc87241b4a2a680139dec02db is 50, key is test_row_0/A:col10/1731836580834/Put/seqid=0 2024-11-17T09:43:00,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742334_1510 (size=19321) 2024-11-17T09:43:00,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:00,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836640905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836640903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:00,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836580898, max timestamp=1731836580918Num uniques 859; 2024-11-17T09:43:00,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836580898, max timestamp=1731836580918Num uniques 859; 2024-11-17T09:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:00,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:00,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:00,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836580898, max timestamp=1731836580918Num uniques 859; 2024-11-17T09:43:00,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T09:43:00,940 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:00,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T09:43:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T09:43:00,942 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:00,942 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:00,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836640993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836640998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836641011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836641030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836641030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T09:43:01,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836641120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836641120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836641120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/731a3adfc87241b4a2a680139dec02db 2024-11-17T09:43:01,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836641240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836641240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T09:43:01,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5518feb7d2be4be2b92796474562c686 is 50, key is test_row_0/B:col10/1731836580834/Put/seqid=0 2024-11-17T09:43:01,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742335_1511 (size=12151) 2024-11-17T09:43:01,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836641330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836641330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836641330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T09:43:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836641569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836641569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836641649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836641649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836641650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:01,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5518feb7d2be4be2b92796474562c686 2024-11-17T09:43:01,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b9617f3097514ac89ccdc1b166d56504 is 50, key is test_row_0/C:col10/1731836580834/Put/seqid=0 2024-11-17T09:43:01,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742336_1512 (size=12151) 2024-11-17T09:43:01,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:01,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b9617f3097514ac89ccdc1b166d56504 2024-11-17T09:43:02,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/731a3adfc87241b4a2a680139dec02db as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/731a3adfc87241b4a2a680139dec02db 2024-11-17T09:43:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T09:43:02,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/731a3adfc87241b4a2a680139dec02db, entries=300, sequenceid=232, filesize=18.9 K 2024-11-17T09:43:02,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/5518feb7d2be4be2b92796474562c686 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5518feb7d2be4be2b92796474562c686 2024-11-17T09:43:02,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5518feb7d2be4be2b92796474562c686, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T09:43:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b9617f3097514ac89ccdc1b166d56504 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b9617f3097514ac89ccdc1b166d56504 2024-11-17T09:43:02,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b9617f3097514ac89ccdc1b166d56504, entries=150, sequenceid=232, filesize=11.9 K 2024-11-17T09:43:02,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fe8ebf75400bab2f775cc2a034eb525d in 1248ms, sequenceid=232, compaction requested=true 2024-11-17T09:43:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:02,093 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T09:43:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:02,093 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 68369 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61199 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/A is initiating minor compaction (all files) 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/B is initiating minor compaction (all files) 2024-11-17T09:43:02,094 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/B in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,094 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/A in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,094 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/84ef08dfc0464a9a96e056f80f239b79, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/76e803a5fa894442a0c1bea44d1561db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eed0a057438447d8d600295241fa84a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/db5d730c671d4455adc8412bfb629c4d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5518feb7d2be4be2b92796474562c686] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=59.8 K 2024-11-17T09:43:02,094 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0488bb7960db47309af94cce3d8ee7ba, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/a84b1a9fe3c845429cab19ef0633c3de, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9ec08a0ae1e2487bb8682bdda49910e9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/775df0bf12e844bd9d0e3323a9b2c0ef, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/731a3adfc87241b4a2a680139dec02db] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=66.8 K 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ef08dfc0464a9a96e056f80f239b79, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731836575371 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0488bb7960db47309af94cce3d8ee7ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731836575371 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84b1a9fe3c845429cab19ef0633c3de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731836575737 2024-11-17T09:43:02,094 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 76e803a5fa894442a0c1bea44d1561db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731836575737 2024-11-17T09:43:02,095 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec08a0ae1e2487bb8682bdda49910e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731836577969 2024-11-17T09:43:02,095 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eed0a057438447d8d600295241fa84a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731836577969 2024-11-17T09:43:02,095 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting db5d730c671d4455adc8412bfb629c4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731836578639 2024-11-17T09:43:02,095 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775df0bf12e844bd9d0e3323a9b2c0ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731836578639 2024-11-17T09:43:02,095 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 5518feb7d2be4be2b92796474562c686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731836580834 2024-11-17T09:43:02,095 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731a3adfc87241b4a2a680139dec02db, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731836580833 2024-11-17T09:43:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:02,106 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:02,107 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6f3f87aa96f14177a16827fd59de7d22 is 50, key is test_row_0/B:col10/1731836580834/Put/seqid=0 2024-11-17T09:43:02,107 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#A#compaction#436 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:02,107 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/4ffe248f226743f291bd32bc446222c2 is 50, key is test_row_0/A:col10/1731836580834/Put/seqid=0 2024-11-17T09:43:02,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-17T09:43:02,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836580958, max timestamp=1731836582102Num uniques 1018; 2024-11-17T09:43:02,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836580958, max timestamp=1731836582102Num uniques 1018; 2024-11-17T09:43:02,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:43:02,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836580958, max timestamp=1731836582102Num uniques 1018; 2024-11-17T09:43:02,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:43:02,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:43:02,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:43:02,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:43:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:43:02,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:02,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742337_1513 (size=12765) 2024-11-17T09:43:02,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742338_1514 (size=12765) 2024-11-17T09:43:02,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/3e77692f6af44b08b37f9b8f584162b6 is 50, key is test_row_0/A:col10/1731836582105/Put/seqid=0 2024-11-17T09:43:02,133 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/4ffe248f226743f291bd32bc446222c2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/4ffe248f226743f291bd32bc446222c2 2024-11-17T09:43:02,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742339_1515 (size=17181) 2024-11-17T09:43:02,140 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/A of fe8ebf75400bab2f775cc2a034eb525d into 4ffe248f226743f291bd32bc446222c2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:02,140 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:02,140 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/A, priority=11, startTime=1731836582092; duration=0sec 2024-11-17T09:43:02,140 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:02,140 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:A 2024-11-17T09:43:02,140 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T09:43:02,141 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61199 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T09:43:02,141 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/C is initiating minor compaction (all files) 2024-11-17T09:43:02,141 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/C in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,141 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/1ab65d4c6fcf41b3812c961a12787ae6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab120298924a43dcbf293d9d1c850ca2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/18c2e09d0a684a11bdfbd1aa3dd594a6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b87d612f3a404499871bc6e75e6912c2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b9617f3097514ac89ccdc1b166d56504] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=59.8 K 2024-11-17T09:43:02,142 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ab65d4c6fcf41b3812c961a12787ae6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731836575371 2024-11-17T09:43:02,142 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab120298924a43dcbf293d9d1c850ca2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731836575737 2024-11-17T09:43:02,142 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c2e09d0a684a11bdfbd1aa3dd594a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731836577969 2024-11-17T09:43:02,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/3e77692f6af44b08b37f9b8f584162b6 2024-11-17T09:43:02,142 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b87d612f3a404499871bc6e75e6912c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731836578639 2024-11-17T09:43:02,143 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9617f3097514ac89ccdc1b166d56504, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731836580834 2024-11-17T09:43:02,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/aa18ecebfa1a43be90c1f750a4b03a73 is 50, key is test_row_0/B:col10/1731836582105/Put/seqid=0 2024-11-17T09:43:02,152 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#C#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:02,153 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/5b6cb06efea44154b5953eb6cb23f491 is 50, key is test_row_0/C:col10/1731836580834/Put/seqid=0 2024-11-17T09:43:02,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742340_1516 (size=12301) 2024-11-17T09:43:02,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742341_1517 (size=12765) 2024-11-17T09:43:02,163 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/5b6cb06efea44154b5953eb6cb23f491 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/5b6cb06efea44154b5953eb6cb23f491 2024-11-17T09:43:02,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,169 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/C of fe8ebf75400bab2f775cc2a034eb525d into 5b6cb06efea44154b5953eb6cb23f491(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:02,169 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:02,169 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/C, priority=11, startTime=1731836582093; duration=0sec 2024-11-17T09:43:02,169 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:02,169 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:C 2024-11-17T09:43:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836582166, max timestamp=1731836582191Num uniques 859; 2024-11-17T09:43:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:43:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836582166, max timestamp=1731836582191Num uniques 859; 2024-11-17T09:43:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:43:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836582166, max timestamp=1731836582191Num uniques 859; 2024-11-17T09:43:02,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:43:02,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731836582201, max timestamp=1731836582235Num uniques 1068; 2024-11-17T09:43:02,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-17T09:43:02,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731836582201, max timestamp=1731836582235Num uniques 1068; 2024-11-17T09:43:02,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731836582201, max timestamp=1731836582235Num uniques 1068; 2024-11-17T09:43:02,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-17T09:43:02,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-17T09:43:02,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836642277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836642283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836642292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836642295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836642296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836642400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836642410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836642410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836642419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836642419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,519 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/6f3f87aa96f14177a16827fd59de7d22 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6f3f87aa96f14177a16827fd59de7d22 2024-11-17T09:43:02,523 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/B of fe8ebf75400bab2f775cc2a034eb525d into 6f3f87aa96f14177a16827fd59de7d22(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:02,523 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:02,523 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/B, priority=11, startTime=1731836582092; duration=0sec 2024-11-17T09:43:02,523 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:02,523 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:B 2024-11-17T09:43:02,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/aa18ecebfa1a43be90c1f750a4b03a73 2024-11-17T09:43:02,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/0414befde6ff409dbacfea67dd75c821 is 50, key is test_row_0/C:col10/1731836582105/Put/seqid=0 2024-11-17T09:43:02,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742342_1518 (size=12301) 2024-11-17T09:43:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836642610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836642630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836642630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836642639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836642639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:02,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:02,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836642920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:02,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:02,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836642950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836642950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836642950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836642950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:02,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/0414befde6ff409dbacfea67dd75c821 2024-11-17T09:43:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/3e77692f6af44b08b37f9b8f584162b6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/3e77692f6af44b08b37f9b8f584162b6 2024-11-17T09:43:02,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/3e77692f6af44b08b37f9b8f584162b6, entries=250, sequenceid=271, filesize=16.8 K 2024-11-17T09:43:02,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/aa18ecebfa1a43be90c1f750a4b03a73 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/aa18ecebfa1a43be90c1f750a4b03a73 2024-11-17T09:43:02,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/aa18ecebfa1a43be90c1f750a4b03a73, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T09:43:02,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/0414befde6ff409dbacfea67dd75c821 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/0414befde6ff409dbacfea67dd75c821 2024-11-17T09:43:02,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/0414befde6ff409dbacfea67dd75c821, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T09:43:02,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=161.02 KB/164880 for fe8ebf75400bab2f775cc2a034eb525d in 874ms, sequenceid=271, compaction requested=false 2024-11-17T09:43:02,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:03,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T09:43:03,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:03,082 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=161.02 KB heapSize=445.08 KB 2024-11-17T09:43:03,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:43:03,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:03,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:43:03,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:03,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:43:03,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/75438901fc3e4453bfa3042d71916a6f is 50, key is test_row_0/A:col10/1731836582274/Put/seqid=0 2024-11-17T09:43:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T09:43:03,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742343_1519 (size=12301) 2024-11-17T09:43:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:03,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. as already flushing 2024-11-17T09:43:03,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836643460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:03,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836643469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:03,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836643469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836643470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836643470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:03,491 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/75438901fc3e4453bfa3042d71916a6f 2024-11-17T09:43:03,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/8c2590e048e5486c9b3172c972a20ac5 is 50, key is test_row_0/B:col10/1731836582274/Put/seqid=0 2024-11-17T09:43:03,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742344_1520 (size=12301) 2024-11-17T09:43:03,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836643580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:03,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836643799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:03,902 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/8c2590e048e5486c9b3172c972a20ac5 2024-11-17T09:43:03,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8189c4e361dd4a35bde3e5a7a4e8f869 is 50, key is test_row_0/C:col10/1731836582274/Put/seqid=0 2024-11-17T09:43:03,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742345_1521 (size=12301) 2024-11-17T09:43:04,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836644110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,313 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8189c4e361dd4a35bde3e5a7a4e8f869 2024-11-17T09:43:04,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/75438901fc3e4453bfa3042d71916a6f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/75438901fc3e4453bfa3042d71916a6f 2024-11-17T09:43:04,320 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/75438901fc3e4453bfa3042d71916a6f, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T09:43:04,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/8c2590e048e5486c9b3172c972a20ac5 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/8c2590e048e5486c9b3172c972a20ac5 2024-11-17T09:43:04,324 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/8c2590e048e5486c9b3172c972a20ac5, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T09:43:04,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/8189c4e361dd4a35bde3e5a7a4e8f869 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8189c4e361dd4a35bde3e5a7a4e8f869 2024-11-17T09:43:04,328 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8189c4e361dd4a35bde3e5a7a4e8f869, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T09:43:04,328 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.03 KB/455712, currentSize=26.84 KB/27480 for fe8ebf75400bab2f775cc2a034eb525d in 1246ms, sequenceid=328, compaction requested=true 2024-11-17T09:43:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T09:43:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T09:43:04,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T09:43:04,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3880 sec 2024-11-17T09:43:04,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 3.3930 sec 2024-11-17T09:43:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:04,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:43:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:43:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:43:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:43:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:04,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/5f858166f2924455b19d663d20ee0b83 is 50, key is test_row_0/A:col10/1731836584493/Put/seqid=0 2024-11-17T09:43:04,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742346_1522 (size=12297) 2024-11-17T09:43:04,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836584522, max timestamp=1731836584535Num uniques 859; 2024-11-17T09:43:04,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:43:04,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836584522, max timestamp=1731836584535Num uniques 859; 2024-11-17T09:43:04,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836584522, max timestamp=1731836584535Num uniques 859; 2024-11-17T09:43:04,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:43:04,539 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:43:04,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836644574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836644575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836644586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836644587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836644630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836644690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,698 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-11-17T09:43:04,698 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:04,698 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,698 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:04,699 INFO [Thread-2071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2071. 2024-11-17T09:43:04,699 DEBUG [Thread-2071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:04,699 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,699 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,699 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836644700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836644700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,708 INFO [Thread-2075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2075. 2024-11-17T09:43:04,708 DEBUG [Thread-2075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:04,708 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,708 DEBUG [Thread-2075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:04,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836644709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,712 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-17T09:43:04,712 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:04,712 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,712 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:04,713 INFO [Thread-2073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2073. 2024-11-17T09:43:04,713 DEBUG [Thread-2073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:04,713 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,713 DEBUG [Thread-2073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:04,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:04,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836644900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/5f858166f2924455b19d663d20ee0b83 2024-11-17T09:43:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836644910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836644910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:04,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/160a06022c2c4895b343443aadb9ad10 is 50, key is test_row_0/B:col10/1731836584493/Put/seqid=0 2024-11-17T09:43:04,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742347_1523 (size=9857) 2024-11-17T09:43:04,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836644919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:04,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T09:43:05,110 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:05,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836645209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836645220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836645220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60226 deadline: 1731836645230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/160a06022c2c4895b343443aadb9ad10 2024-11-17T09:43:05,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/417bb2779b9d4a2a8b70b7c0886b16f7 is 50, key is test_row_0/C:col10/1731836584493/Put/seqid=0 2024-11-17T09:43:05,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742348_1524 (size=9857) 2024-11-17T09:43:05,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60202 deadline: 1731836645650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,651 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in fe8ebf75400bab2f775cc2a034eb525d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60266 deadline: 1731836645719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/417bb2779b9d4a2a8b70b7c0886b16f7 2024-11-17T09:43:05,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/5f858166f2924455b19d663d20ee0b83 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5f858166f2924455b19d663d20ee0b83 2024-11-17T09:43:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60232 deadline: 1731836645730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1731836645730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 , the old value is region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:05,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., hostname=70e37c475432,40671,1731836432285, seqNum=2 because the exception is null or not the one we care about 2024-11-17T09:43:05,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5f858166f2924455b19d663d20ee0b83, entries=150, sequenceid=338, filesize=12.0 K 2024-11-17T09:43:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/160a06022c2c4895b343443aadb9ad10 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/160a06022c2c4895b343443aadb9ad10 2024-11-17T09:43:05,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/160a06022c2c4895b343443aadb9ad10, entries=100, sequenceid=338, filesize=9.6 K 2024-11-17T09:43:05,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/417bb2779b9d4a2a8b70b7c0886b16f7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/417bb2779b9d4a2a8b70b7c0886b16f7 2024-11-17T09:43:05,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/417bb2779b9d4a2a8b70b7c0886b16f7, entries=100, sequenceid=338, filesize=9.6 K 2024-11-17T09:43:05,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fe8ebf75400bab2f775cc2a034eb525d in 1242ms, sequenceid=338, compaction requested=true 2024-11-17T09:43:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe8ebf75400bab2f775cc2a034eb525d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:05,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:05,739 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:43:05,739 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:43:05,741 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47224 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:43:05,741 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:43:05,741 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/B is initiating minor compaction (all files) 2024-11-17T09:43:05,741 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/A is initiating minor compaction (all files) 2024-11-17T09:43:05,741 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/B in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:05,741 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/A in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:05,741 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6f3f87aa96f14177a16827fd59de7d22, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/aa18ecebfa1a43be90c1f750a4b03a73, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/8c2590e048e5486c9b3172c972a20ac5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/160a06022c2c4895b343443aadb9ad10] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=46.1 K 2024-11-17T09:43:05,741 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/4ffe248f226743f291bd32bc446222c2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/3e77692f6af44b08b37f9b8f584162b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/75438901fc3e4453bfa3042d71916a6f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5f858166f2924455b19d663d20ee0b83] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=53.3 K 2024-11-17T09:43:05,741 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3f87aa96f14177a16827fd59de7d22, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731836580834 2024-11-17T09:43:05,741 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffe248f226743f291bd32bc446222c2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731836580834 2024-11-17T09:43:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:05,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-17T09:43:05,741 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting aa18ecebfa1a43be90c1f750a4b03a73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731836580958 2024-11-17T09:43:05,742 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e77692f6af44b08b37f9b8f584162b6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731836580958 2024-11-17T09:43:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:43:05,742 INFO [Thread-2062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2062. 2024-11-17T09:43:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:43:05,742 DEBUG [Thread-2062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:05,742 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c2590e048e5486c9b3172c972a20ac5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731836582274 2024-11-17T09:43:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:43:05,742 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:05,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:05,742 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75438901fc3e4453bfa3042d71916a6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731836582274 2024-11-17T09:43:05,742 DEBUG [Thread-2062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:05,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:05,742 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 160a06022c2c4895b343443aadb9ad10, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731836584493 2024-11-17T09:43:05,742 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f858166f2924455b19d663d20ee0b83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731836584493 2024-11-17T09:43:05,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/7e985fa63bf743578fa436b3107a7d5f is 50, key is test_row_0/A:col10/1731836584584/Put/seqid=0 2024-11-17T09:43:05,750 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:05,750 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/1c1cbf95f1d14c2482fc43bd41aa7e3f is 50, key is test_row_0/B:col10/1731836584493/Put/seqid=0 2024-11-17T09:43:05,752 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#A#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:05,753 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/9e7fa235ea58415c84a06e928e66faee is 50, key is test_row_0/A:col10/1731836584493/Put/seqid=0 2024-11-17T09:43:05,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742349_1525 (size=12301) 2024-11-17T09:43:05,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742350_1526 (size=13051) 2024-11-17T09:43:05,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742351_1527 (size=13051) 2024-11-17T09:43:05,759 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/9e7fa235ea58415c84a06e928e66faee as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9e7fa235ea58415c84a06e928e66faee 2024-11-17T09:43:05,763 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/A of fe8ebf75400bab2f775cc2a034eb525d into 9e7fa235ea58415c84a06e928e66faee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:05,763 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:05,763 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/A, priority=12, startTime=1731836585739; duration=0sec 2024-11-17T09:43:05,763 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:05,763 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:A 2024-11-17T09:43:05,763 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:43:05,764 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47224 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:43:05,764 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): fe8ebf75400bab2f775cc2a034eb525d/C is initiating minor compaction (all files) 2024-11-17T09:43:05,764 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe8ebf75400bab2f775cc2a034eb525d/C in TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:05,764 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/5b6cb06efea44154b5953eb6cb23f491, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/0414befde6ff409dbacfea67dd75c821, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8189c4e361dd4a35bde3e5a7a4e8f869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/417bb2779b9d4a2a8b70b7c0886b16f7] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp, totalSize=46.1 K 2024-11-17T09:43:05,765 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6cb06efea44154b5953eb6cb23f491, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731836580834 2024-11-17T09:43:05,765 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0414befde6ff409dbacfea67dd75c821, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731836580958 2024-11-17T09:43:05,765 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8189c4e361dd4a35bde3e5a7a4e8f869, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731836582274 2024-11-17T09:43:05,765 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417bb2779b9d4a2a8b70b7c0886b16f7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731836584493 2024-11-17T09:43:05,771 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe8ebf75400bab2f775cc2a034eb525d#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:05,771 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/c842730dab704a3b8a364ddb151bbc72 is 50, key is test_row_0/C:col10/1731836584493/Put/seqid=0 2024-11-17T09:43:05,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742352_1528 (size=13051) 2024-11-17T09:43:05,777 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/c842730dab704a3b8a364ddb151bbc72 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c842730dab704a3b8a364ddb151bbc72 2024-11-17T09:43:05,781 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/C of fe8ebf75400bab2f775cc2a034eb525d into c842730dab704a3b8a364ddb151bbc72(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:05,781 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:05,781 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/C, priority=12, startTime=1731836585739; duration=0sec 2024-11-17T09:43:05,781 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:05,781 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:C 2024-11-17T09:43:06,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/7e985fa63bf743578fa436b3107a7d5f 2024-11-17T09:43:06,158 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/1c1cbf95f1d14c2482fc43bd41aa7e3f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1c1cbf95f1d14c2482fc43bd41aa7e3f 2024-11-17T09:43:06,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/576e30022b6a4655bf9ac6076fa7fd6c is 50, key is test_row_0/B:col10/1731836584584/Put/seqid=0 2024-11-17T09:43:06,162 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fe8ebf75400bab2f775cc2a034eb525d/B of fe8ebf75400bab2f775cc2a034eb525d into 1c1cbf95f1d14c2482fc43bd41aa7e3f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:06,162 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:06,162 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., storeName=fe8ebf75400bab2f775cc2a034eb525d/B, priority=12, startTime=1731836585739; duration=0sec 2024-11-17T09:43:06,162 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:06,162 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe8ebf75400bab2f775cc2a034eb525d:B 2024-11-17T09:43:06,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742353_1529 (size=12301) 2024-11-17T09:43:06,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/576e30022b6a4655bf9ac6076fa7fd6c 2024-11-17T09:43:06,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b661540a8ce14481ab378d8f59aa40d8 is 50, key is test_row_0/C:col10/1731836584584/Put/seqid=0 2024-11-17T09:43:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742354_1530 (size=12301) 2024-11-17T09:43:06,731 INFO [Thread-2064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2064. 2024-11-17T09:43:06,731 DEBUG [Thread-2064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:06,731 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:06,731 DEBUG [Thread-2064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:06,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:06,741 INFO [Thread-2056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2056. 2024-11-17T09:43:06,741 INFO [Thread-2060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2060. 2024-11-17T09:43:06,741 DEBUG [Thread-2060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:06,741 DEBUG [Thread-2056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:06,741 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:06,741 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:06,741 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:06,741 DEBUG [Thread-2060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:06,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:06,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:06,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b661540a8ce14481ab378d8f59aa40d8 2024-11-17T09:43:06,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/7e985fa63bf743578fa436b3107a7d5f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/7e985fa63bf743578fa436b3107a7d5f 2024-11-17T09:43:06,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/7e985fa63bf743578fa436b3107a7d5f, entries=150, sequenceid=374, filesize=12.0 K 2024-11-17T09:43:06,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/576e30022b6a4655bf9ac6076fa7fd6c as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/576e30022b6a4655bf9ac6076fa7fd6c 2024-11-17T09:43:06,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/576e30022b6a4655bf9ac6076fa7fd6c, entries=150, sequenceid=374, filesize=12.0 K 2024-11-17T09:43:06,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/b661540a8ce14481ab378d8f59aa40d8 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b661540a8ce14481ab378d8f59aa40d8 2024-11-17T09:43:06,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b661540a8ce14481ab378d8f59aa40d8, entries=150, sequenceid=374, filesize=12.0 K 2024-11-17T09:43:06,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=20.13 KB/20610 for fe8ebf75400bab2f775cc2a034eb525d in 1245ms, sequenceid=374, compaction requested=false 2024-11-17T09:43:06,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe8ebf75400bab2f775cc2a034eb525d: 2024-11-17T09:43:07,661 INFO [Thread-2058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2058. 2024-11-17T09:43:07,661 DEBUG [Thread-2058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:07,661 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:07,662 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:07,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1105 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3311 rows 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1105 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3315 rows 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1108 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3324 rows 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1112 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3332 rows 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1112 2024-11-17T09:43:07,662 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3335 rows 2024-11-17T09:43:07,662 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:43:07,662 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:43:07,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:07,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:07,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:07,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T09:43:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T09:43:07,666 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836587665"}]},"ts":"1731836587665"} 2024-11-17T09:43:07,667 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T09:43:07,667 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T09:43:07,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:43:07,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, UNASSIGN}] 2024-11-17T09:43:07,669 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, UNASSIGN 2024-11-17T09:43:07,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fe8ebf75400bab2f775cc2a034eb525d, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:43:07,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, UNASSIGN because future has completed 2024-11-17T09:43:07,671 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:43:07,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:43:07,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T09:43:07,823 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing fe8ebf75400bab2f775cc2a034eb525d, disabling compactions & flushes 2024-11-17T09:43:07,823 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. after waiting 0 ms 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:07,823 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing fe8ebf75400bab2f775cc2a034eb525d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=A 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=B 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe8ebf75400bab2f775cc2a034eb525d, store=C 2024-11-17T09:43:07,823 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:07,827 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/bc4369f982d9407a9569e5635b310c49 is 50, key is test_row_1/A:col10/1731836587660/Put/seqid=0 2024-11-17T09:43:07,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742355_1531 (size=9857) 2024-11-17T09:43:07,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T09:43:08,230 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/bc4369f982d9407a9569e5635b310c49 2024-11-17T09:43:08,236 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/f7e98ce3b50841f4af70bb9212568181 is 50, key is test_row_1/B:col10/1731836587660/Put/seqid=0 2024-11-17T09:43:08,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742356_1532 (size=9857) 2024-11-17T09:43:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T09:43:08,639 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/f7e98ce3b50841f4af70bb9212568181 2024-11-17T09:43:08,645 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/bdcc5c725e734e5aa21b4d3aec907f38 is 50, key is test_row_1/C:col10/1731836587660/Put/seqid=0 2024-11-17T09:43:08,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742357_1533 (size=9857) 2024-11-17T09:43:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T09:43:09,049 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/bdcc5c725e734e5aa21b4d3aec907f38 2024-11-17T09:43:09,054 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/A/bc4369f982d9407a9569e5635b310c49 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc4369f982d9407a9569e5635b310c49 2024-11-17T09:43:09,058 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc4369f982d9407a9569e5635b310c49, entries=100, sequenceid=384, filesize=9.6 K 2024-11-17T09:43:09,059 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/B/f7e98ce3b50841f4af70bb9212568181 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/f7e98ce3b50841f4af70bb9212568181 2024-11-17T09:43:09,062 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/f7e98ce3b50841f4af70bb9212568181, entries=100, sequenceid=384, filesize=9.6 K 2024-11-17T09:43:09,063 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/.tmp/C/bdcc5c725e734e5aa21b4d3aec907f38 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/bdcc5c725e734e5aa21b4d3aec907f38 2024-11-17T09:43:09,067 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/bdcc5c725e734e5aa21b4d3aec907f38, entries=100, sequenceid=384, filesize=9.6 K 2024-11-17T09:43:09,068 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fe8ebf75400bab2f775cc2a034eb525d in 1245ms, sequenceid=384, compaction requested=true 2024-11-17T09:43:09,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/827c44f0fd4e4314aa3c44d8d5488398, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/07192f504874430584af3c56410ef9b4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/b6063a3c8539453198cc0734c5f7c42c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fd36bd364ef1464585765497f22cd11f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/735e1e6ea0ad430f930b8a12e9748616, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0878b8f3098145f9bd5097c229c64930, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/91fe9283bacd4517a19c03073ee19420, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5389ef44df7b4bcfa12bd3fe7a51f7ce, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc37d17a0a16466dbb9bf91bb8b7e4a5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/47f6a606b45c4820a8e85b3ff77e35cd, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fb89c6260e844c758fe6cfc4da8c0d77, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/f5c1a2dc927d4577bca599410e605c2e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/e1f13af01ed94c85adfc186be4faa671, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0488bb7960db47309af94cce3d8ee7ba, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/a84b1a9fe3c845429cab19ef0633c3de, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9ec08a0ae1e2487bb8682bdda49910e9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/775df0bf12e844bd9d0e3323a9b2c0ef, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/731a3adfc87241b4a2a680139dec02db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/4ffe248f226743f291bd32bc446222c2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/3e77692f6af44b08b37f9b8f584162b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/75438901fc3e4453bfa3042d71916a6f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5f858166f2924455b19d663d20ee0b83] to archive 2024-11-17T09:43:09,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:43:09,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/827c44f0fd4e4314aa3c44d8d5488398 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/827c44f0fd4e4314aa3c44d8d5488398 2024-11-17T09:43:09,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/07192f504874430584af3c56410ef9b4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/07192f504874430584af3c56410ef9b4 2024-11-17T09:43:09,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/b6063a3c8539453198cc0734c5f7c42c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/b6063a3c8539453198cc0734c5f7c42c 2024-11-17T09:43:09,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fd36bd364ef1464585765497f22cd11f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fd36bd364ef1464585765497f22cd11f 2024-11-17T09:43:09,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/735e1e6ea0ad430f930b8a12e9748616 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/735e1e6ea0ad430f930b8a12e9748616 2024-11-17T09:43:09,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0878b8f3098145f9bd5097c229c64930 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0878b8f3098145f9bd5097c229c64930 2024-11-17T09:43:09,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/91fe9283bacd4517a19c03073ee19420 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/91fe9283bacd4517a19c03073ee19420 2024-11-17T09:43:09,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5389ef44df7b4bcfa12bd3fe7a51f7ce to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5389ef44df7b4bcfa12bd3fe7a51f7ce 2024-11-17T09:43:09,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc37d17a0a16466dbb9bf91bb8b7e4a5 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc37d17a0a16466dbb9bf91bb8b7e4a5 2024-11-17T09:43:09,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/47f6a606b45c4820a8e85b3ff77e35cd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/47f6a606b45c4820a8e85b3ff77e35cd 2024-11-17T09:43:09,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fb89c6260e844c758fe6cfc4da8c0d77 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/fb89c6260e844c758fe6cfc4da8c0d77 2024-11-17T09:43:09,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/f5c1a2dc927d4577bca599410e605c2e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/f5c1a2dc927d4577bca599410e605c2e 2024-11-17T09:43:09,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/e1f13af01ed94c85adfc186be4faa671 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/e1f13af01ed94c85adfc186be4faa671 2024-11-17T09:43:09,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0488bb7960db47309af94cce3d8ee7ba to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/0488bb7960db47309af94cce3d8ee7ba 2024-11-17T09:43:09,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/a84b1a9fe3c845429cab19ef0633c3de to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/a84b1a9fe3c845429cab19ef0633c3de 2024-11-17T09:43:09,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9ec08a0ae1e2487bb8682bdda49910e9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9ec08a0ae1e2487bb8682bdda49910e9 2024-11-17T09:43:09,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/775df0bf12e844bd9d0e3323a9b2c0ef to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/775df0bf12e844bd9d0e3323a9b2c0ef 2024-11-17T09:43:09,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/731a3adfc87241b4a2a680139dec02db to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/731a3adfc87241b4a2a680139dec02db 2024-11-17T09:43:09,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/4ffe248f226743f291bd32bc446222c2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/4ffe248f226743f291bd32bc446222c2 2024-11-17T09:43:09,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/3e77692f6af44b08b37f9b8f584162b6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/3e77692f6af44b08b37f9b8f584162b6 2024-11-17T09:43:09,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/75438901fc3e4453bfa3042d71916a6f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/75438901fc3e4453bfa3042d71916a6f 2024-11-17T09:43:09,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5f858166f2924455b19d663d20ee0b83 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/5f858166f2924455b19d663d20ee0b83 2024-11-17T09:43:09,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/505d4e76fc3b4bfcbf65221c617b90c5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/0f0e3bc501854019b41bb6aa7f6a4194, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/c4e0b59bb6f040bcab3cc73ee6e37c87, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/ee7b423b202a4f8692dc4920a9348159, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5a2a71c6f61645709b8bc8ce821f38a6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a7fdcc8f2abb4f87bafd4fdafdd907c1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1128c31d4c054107a92ff1198cc3d3ec, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/479b31cc63464a4db9af78b85ef69ef1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a0825c76b6eb49cf8d2f2e5bae29bd1e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eefe3a5556a4b1faa4a69b014ba482b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c1235ec7e1449a89e1d8bf63eb5bf06, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/61e0549f3d7f4d96ba6f397c7daf359b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/84ef08dfc0464a9a96e056f80f239b79, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c749be6e6c44ab689c5c86d3fe5dc41, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/76e803a5fa894442a0c1bea44d1561db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eed0a057438447d8d600295241fa84a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/db5d730c671d4455adc8412bfb629c4d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6f3f87aa96f14177a16827fd59de7d22, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5518feb7d2be4be2b92796474562c686, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/aa18ecebfa1a43be90c1f750a4b03a73, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/8c2590e048e5486c9b3172c972a20ac5, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/160a06022c2c4895b343443aadb9ad10] to archive 2024-11-17T09:43:09,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:43:09,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/505d4e76fc3b4bfcbf65221c617b90c5 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/505d4e76fc3b4bfcbf65221c617b90c5 2024-11-17T09:43:09,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/0f0e3bc501854019b41bb6aa7f6a4194 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/0f0e3bc501854019b41bb6aa7f6a4194 2024-11-17T09:43:09,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/c4e0b59bb6f040bcab3cc73ee6e37c87 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/c4e0b59bb6f040bcab3cc73ee6e37c87 2024-11-17T09:43:09,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/ee7b423b202a4f8692dc4920a9348159 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/ee7b423b202a4f8692dc4920a9348159 2024-11-17T09:43:09,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5a2a71c6f61645709b8bc8ce821f38a6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5a2a71c6f61645709b8bc8ce821f38a6 2024-11-17T09:43:09,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a7fdcc8f2abb4f87bafd4fdafdd907c1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a7fdcc8f2abb4f87bafd4fdafdd907c1 2024-11-17T09:43:09,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1128c31d4c054107a92ff1198cc3d3ec to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1128c31d4c054107a92ff1198cc3d3ec 2024-11-17T09:43:09,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/479b31cc63464a4db9af78b85ef69ef1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/479b31cc63464a4db9af78b85ef69ef1 2024-11-17T09:43:09,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a0825c76b6eb49cf8d2f2e5bae29bd1e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/a0825c76b6eb49cf8d2f2e5bae29bd1e 2024-11-17T09:43:09,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eefe3a5556a4b1faa4a69b014ba482b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eefe3a5556a4b1faa4a69b014ba482b 2024-11-17T09:43:09,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c1235ec7e1449a89e1d8bf63eb5bf06 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c1235ec7e1449a89e1d8bf63eb5bf06 2024-11-17T09:43:09,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/61e0549f3d7f4d96ba6f397c7daf359b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/61e0549f3d7f4d96ba6f397c7daf359b 2024-11-17T09:43:09,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/84ef08dfc0464a9a96e056f80f239b79 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/84ef08dfc0464a9a96e056f80f239b79 2024-11-17T09:43:09,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c749be6e6c44ab689c5c86d3fe5dc41 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6c749be6e6c44ab689c5c86d3fe5dc41 2024-11-17T09:43:09,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/76e803a5fa894442a0c1bea44d1561db to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/76e803a5fa894442a0c1bea44d1561db 2024-11-17T09:43:09,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eed0a057438447d8d600295241fa84a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5eed0a057438447d8d600295241fa84a 2024-11-17T09:43:09,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/db5d730c671d4455adc8412bfb629c4d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/db5d730c671d4455adc8412bfb629c4d 2024-11-17T09:43:09,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6f3f87aa96f14177a16827fd59de7d22 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/6f3f87aa96f14177a16827fd59de7d22 2024-11-17T09:43:09,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5518feb7d2be4be2b92796474562c686 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/5518feb7d2be4be2b92796474562c686 2024-11-17T09:43:09,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/aa18ecebfa1a43be90c1f750a4b03a73 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/aa18ecebfa1a43be90c1f750a4b03a73 2024-11-17T09:43:09,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/8c2590e048e5486c9b3172c972a20ac5 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/8c2590e048e5486c9b3172c972a20ac5 2024-11-17T09:43:09,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/160a06022c2c4895b343443aadb9ad10 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/160a06022c2c4895b343443aadb9ad10 2024-11-17T09:43:09,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d141fcb365e845c7bc330c6814d03d98, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab4eaab7339140ad90dd1b6c206fbc95, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/394b687f2b2142978fa175c8c983a463, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/78a441422d7640dc8a63311ee1e508ac, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8648aa1919e3461894980d04c2192c88, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d6a2cfed42934a19b5922bfc0452e87d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/968a924377e349d19d682c20d36481ac, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8f12bf55698e490c9cb08e1dccb7417c, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c4585969687d4e4ebc687989a3c1dfe1, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/08bbfae918fa4c128ef60423383f237b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/eee8f4b345914db4a282d35fcc882870, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/f7718f83cc8744bea13da1ffabbe9f25, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/1ab65d4c6fcf41b3812c961a12787ae6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8d0bcb0f83a44b6093731f2e27940b7b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab120298924a43dcbf293d9d1c850ca2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/18c2e09d0a684a11bdfbd1aa3dd594a6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b87d612f3a404499871bc6e75e6912c2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/5b6cb06efea44154b5953eb6cb23f491, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b9617f3097514ac89ccdc1b166d56504, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/0414befde6ff409dbacfea67dd75c821, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8189c4e361dd4a35bde3e5a7a4e8f869, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/417bb2779b9d4a2a8b70b7c0886b16f7] to archive 2024-11-17T09:43:09,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:43:09,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d141fcb365e845c7bc330c6814d03d98 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d141fcb365e845c7bc330c6814d03d98 2024-11-17T09:43:09,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab4eaab7339140ad90dd1b6c206fbc95 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab4eaab7339140ad90dd1b6c206fbc95 2024-11-17T09:43:09,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/394b687f2b2142978fa175c8c983a463 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/394b687f2b2142978fa175c8c983a463 2024-11-17T09:43:09,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/78a441422d7640dc8a63311ee1e508ac to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/78a441422d7640dc8a63311ee1e508ac 2024-11-17T09:43:09,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8648aa1919e3461894980d04c2192c88 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8648aa1919e3461894980d04c2192c88 2024-11-17T09:43:09,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d6a2cfed42934a19b5922bfc0452e87d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/d6a2cfed42934a19b5922bfc0452e87d 2024-11-17T09:43:09,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/968a924377e349d19d682c20d36481ac to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/968a924377e349d19d682c20d36481ac 2024-11-17T09:43:09,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8f12bf55698e490c9cb08e1dccb7417c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8f12bf55698e490c9cb08e1dccb7417c 2024-11-17T09:43:09,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c4585969687d4e4ebc687989a3c1dfe1 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c4585969687d4e4ebc687989a3c1dfe1 2024-11-17T09:43:09,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/08bbfae918fa4c128ef60423383f237b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/08bbfae918fa4c128ef60423383f237b 2024-11-17T09:43:09,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/eee8f4b345914db4a282d35fcc882870 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/eee8f4b345914db4a282d35fcc882870 2024-11-17T09:43:09,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/f7718f83cc8744bea13da1ffabbe9f25 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/f7718f83cc8744bea13da1ffabbe9f25 2024-11-17T09:43:09,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/1ab65d4c6fcf41b3812c961a12787ae6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/1ab65d4c6fcf41b3812c961a12787ae6 2024-11-17T09:43:09,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8d0bcb0f83a44b6093731f2e27940b7b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8d0bcb0f83a44b6093731f2e27940b7b 2024-11-17T09:43:09,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab120298924a43dcbf293d9d1c850ca2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/ab120298924a43dcbf293d9d1c850ca2 2024-11-17T09:43:09,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/18c2e09d0a684a11bdfbd1aa3dd594a6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/18c2e09d0a684a11bdfbd1aa3dd594a6 2024-11-17T09:43:09,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b87d612f3a404499871bc6e75e6912c2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b87d612f3a404499871bc6e75e6912c2 2024-11-17T09:43:09,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/5b6cb06efea44154b5953eb6cb23f491 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/5b6cb06efea44154b5953eb6cb23f491 2024-11-17T09:43:09,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b9617f3097514ac89ccdc1b166d56504 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b9617f3097514ac89ccdc1b166d56504 2024-11-17T09:43:09,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/0414befde6ff409dbacfea67dd75c821 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/0414befde6ff409dbacfea67dd75c821 2024-11-17T09:43:09,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8189c4e361dd4a35bde3e5a7a4e8f869 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/8189c4e361dd4a35bde3e5a7a4e8f869 2024-11-17T09:43:09,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/417bb2779b9d4a2a8b70b7c0886b16f7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/417bb2779b9d4a2a8b70b7c0886b16f7 2024-11-17T09:43:09,134 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/recovered.edits/387.seqid, newMaxSeqId=387, maxSeqId=1 2024-11-17T09:43:09,134 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. 2024-11-17T09:43:09,134 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for fe8ebf75400bab2f775cc2a034eb525d: Waiting for close lock at 1731836587823Running coprocessor pre-close hooks at 1731836587823Disabling compacts and flushes for region at 1731836587823Disabling writes for close at 1731836587823Obtaining lock to block concurrent updates at 1731836587823Preparing flush snapshotting stores in fe8ebf75400bab2f775cc2a034eb525d at 1731836587823Finished memstore snapshotting TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731836587824 (+1 ms)Flushing stores of TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d. at 1731836587824Flushing fe8ebf75400bab2f775cc2a034eb525d/A: creating writer at 1731836587824Flushing fe8ebf75400bab2f775cc2a034eb525d/A: appending metadata at 1731836587826 (+2 ms)Flushing fe8ebf75400bab2f775cc2a034eb525d/A: closing flushed file at 1731836587826Flushing fe8ebf75400bab2f775cc2a034eb525d/B: creating writer at 1731836588233 (+407 ms)Flushing fe8ebf75400bab2f775cc2a034eb525d/B: appending metadata at 1731836588235 (+2 ms)Flushing fe8ebf75400bab2f775cc2a034eb525d/B: closing flushed file at 1731836588235Flushing fe8ebf75400bab2f775cc2a034eb525d/C: creating writer at 1731836588642 (+407 ms)Flushing fe8ebf75400bab2f775cc2a034eb525d/C: appending metadata at 1731836588644 (+2 ms)Flushing fe8ebf75400bab2f775cc2a034eb525d/C: closing flushed file at 1731836588644Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f0cd01e: reopening flushed file at 1731836589053 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f5975ef: reopening flushed file at 1731836589058 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cc18e19: reopening flushed file at 1731836589063 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for fe8ebf75400bab2f775cc2a034eb525d in 1245ms, sequenceid=384, compaction requested=true at 1731836589068 (+5 ms)Writing region close event to WAL at 1731836589131 (+63 ms)Running coprocessor post-close hooks at 1731836589134 (+3 ms)Closed at 1731836589134 2024-11-17T09:43:09,136 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:09,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=fe8ebf75400bab2f775cc2a034eb525d, regionState=CLOSED 2024-11-17T09:43:09,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:43:09,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T09:43:09,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure fe8ebf75400bab2f775cc2a034eb525d, server=70e37c475432,40671,1731836432285 in 1.4670 sec 2024-11-17T09:43:09,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-17T09:43:09,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe8ebf75400bab2f775cc2a034eb525d, UNASSIGN in 1.4710 sec 2024-11-17T09:43:09,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T09:43:09,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-17T09:43:09,143 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836589143"}]},"ts":"1731836589143"} 2024-11-17T09:43:09,144 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T09:43:09,144 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T09:43:09,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4820 sec 2024-11-17T09:43:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T09:43:09,810 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:09,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T09:43:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:09,812 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T09:43:09,814 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:09,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:09,817 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/recovered.edits] 2024-11-17T09:43:09,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/7e985fa63bf743578fa436b3107a7d5f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/7e985fa63bf743578fa436b3107a7d5f 2024-11-17T09:43:09,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9e7fa235ea58415c84a06e928e66faee to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/9e7fa235ea58415c84a06e928e66faee 2024-11-17T09:43:09,821 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc4369f982d9407a9569e5635b310c49 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/A/bc4369f982d9407a9569e5635b310c49 2024-11-17T09:43:09,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1c1cbf95f1d14c2482fc43bd41aa7e3f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/1c1cbf95f1d14c2482fc43bd41aa7e3f 2024-11-17T09:43:09,824 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/576e30022b6a4655bf9ac6076fa7fd6c to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/576e30022b6a4655bf9ac6076fa7fd6c 2024-11-17T09:43:09,825 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/f7e98ce3b50841f4af70bb9212568181 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/B/f7e98ce3b50841f4af70bb9212568181 2024-11-17T09:43:09,828 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b661540a8ce14481ab378d8f59aa40d8 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/b661540a8ce14481ab378d8f59aa40d8 2024-11-17T09:43:09,829 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/bdcc5c725e734e5aa21b4d3aec907f38 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/bdcc5c725e734e5aa21b4d3aec907f38 2024-11-17T09:43:09,830 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c842730dab704a3b8a364ddb151bbc72 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/C/c842730dab704a3b8a364ddb151bbc72 2024-11-17T09:43:09,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/recovered.edits/387.seqid to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d/recovered.edits/387.seqid 2024-11-17T09:43:09,833 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/fe8ebf75400bab2f775cc2a034eb525d 2024-11-17T09:43:09,833 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T09:43:09,835 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:09,837 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T09:43:09,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T09:43:09,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:09,840 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T09:43:09,840 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731836589840"}]},"ts":"9223372036854775807"} 2024-11-17T09:43:09,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T09:43:09,841 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fe8ebf75400bab2f775cc2a034eb525d, NAME => 'TestAcidGuarantees,,1731836562510.fe8ebf75400bab2f775cc2a034eb525d.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T09:43:09,842 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T09:43:09,842 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731836589842"}]},"ts":"9223372036854775807"} 2024-11-17T09:43:09,843 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T09:43:09,843 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:09,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-17T09:43:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T09:43:09,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T09:43:09,920 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:09,930 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 247) - Thread LEAK? -, OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 361) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2324 (was 2349) 2024-11-17T09:43:09,941 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=2323 2024-11-17T09:43:09,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:43:09,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:43:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:09,944 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:43:09,944 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:09,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-17T09:43:09,944 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:43:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T09:43:09,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742358_1534 (size=1061) 2024-11-17T09:43:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T09:43:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T09:43:10,352 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be 2024-11-17T09:43:10,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742359_1535 (size=53) 2024-11-17T09:43:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T09:43:10,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:43:10,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0b08a5bb92e2e35ac961ce00a4489f23, disabling compactions & flushes 2024-11-17T09:43:10,759 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:10,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:10,760 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. after waiting 0 ms 2024-11-17T09:43:10,760 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:10,760 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:10,760 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0b08a5bb92e2e35ac961ce00a4489f23: Waiting for close lock at 1731836590759Disabling compacts and flushes for region at 1731836590759Disabling writes for close at 1731836590760 (+1 ms)Writing region close event to WAL at 1731836590760Closed at 1731836590760 2024-11-17T09:43:10,761 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:43:10,761 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731836590761"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836590761"}]},"ts":"1731836590761"} 2024-11-17T09:43:10,763 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:43:10,764 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:43:10,764 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836590764"}]},"ts":"1731836590764"} 2024-11-17T09:43:10,766 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T09:43:10,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, ASSIGN}] 2024-11-17T09:43:10,769 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, ASSIGN 2024-11-17T09:43:10,770 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, ASSIGN; state=OFFLINE, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=false 2024-11-17T09:43:10,921 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:43:10,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, ASSIGN because future has completed 2024-11-17T09:43:10,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:43:11,077 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:11,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:43:11,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:43:11,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,077 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,078 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,079 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:43:11,079 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b08a5bb92e2e35ac961ce00a4489f23 columnFamilyName A 2024-11-17T09:43:11,079 DEBUG [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:11,080 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(327): Store=0b08a5bb92e2e35ac961ce00a4489f23/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:43:11,080 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T09:43:11,081 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:43:11,081 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b08a5bb92e2e35ac961ce00a4489f23 columnFamilyName B 2024-11-17T09:43:11,081 DEBUG [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:11,081 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(327): Store=0b08a5bb92e2e35ac961ce00a4489f23/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:43:11,081 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,082 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:43:11,082 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b08a5bb92e2e35ac961ce00a4489f23 columnFamilyName C 2024-11-17T09:43:11,082 DEBUG [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:11,082 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(327): Store=0b08a5bb92e2e35ac961ce00a4489f23/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:43:11,083 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:11,083 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,083 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,083 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,084 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,084 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,085 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:43:11,086 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,087 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:43:11,088 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 0b08a5bb92e2e35ac961ce00a4489f23; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68894964, jitterRate=0.0266149640083313}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:43:11,088 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:11,088 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 0b08a5bb92e2e35ac961ce00a4489f23: Running coprocessor pre-open hook at 1731836591077Writing region info on filesystem at 1731836591077Initializing all the Stores at 1731836591078 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836591078Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836591078Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836591078Cleaning up temporary data from old regions at 1731836591084 (+6 ms)Running coprocessor post-open hooks at 1731836591088 (+4 ms)Region opened successfully at 1731836591088 2024-11-17T09:43:11,089 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., pid=142, masterSystemTime=1731836591074 2024-11-17T09:43:11,090 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:11,090 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:11,091 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:43:11,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:43:11,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-17T09:43:11,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 in 171 msec 2024-11-17T09:43:11,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-17T09:43:11,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, ASSIGN in 328 msec 2024-11-17T09:43:11,096 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:43:11,096 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836591096"}]},"ts":"1731836591096"} 2024-11-17T09:43:11,097 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T09:43:11,098 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:43:11,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-11-17T09:43:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-17T09:43:12,090 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:12,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:12,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:12,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:12,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:12,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:12,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:12,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585dde7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:12,093 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:12,094 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:12,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421a2277, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:12,094 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:12,096 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:12,097 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:43:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,44533,1731836431485 2024-11-17T09:43:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@e4a1e24 2024-11-17T09:43:12,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:43:12,098 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:43:12,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T09:43:12,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:43:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:12,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742360_1536 (size=1097) 2024-11-17T09:43:12,509 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T09:43:12,509 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T09:43:12,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:43:12,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, REOPEN/MOVE}] 2024-11-17T09:43:12,513 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, REOPEN/MOVE 2024-11-17T09:43:12,514 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:43:12,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, REOPEN/MOVE because future has completed 2024-11-17T09:43:12,515 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:43:12,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:43:12,667 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,667 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:43:12,667 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 0b08a5bb92e2e35ac961ce00a4489f23, disabling compactions & flushes 2024-11-17T09:43:12,667 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,667 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,667 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. after waiting 0 ms 2024-11-17T09:43:12,668 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,671 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T09:43:12,671 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,671 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 0b08a5bb92e2e35ac961ce00a4489f23: Waiting for close lock at 1731836592667Running coprocessor pre-close hooks at 1731836592667Disabling compacts and flushes for region at 1731836592667Disabling writes for close at 1731836592667Writing region close event to WAL at 1731836592668 (+1 ms)Running coprocessor post-close hooks at 1731836592671 (+3 ms)Closed at 1731836592671 2024-11-17T09:43:12,671 WARN [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 0b08a5bb92e2e35ac961ce00a4489f23 to self. 2024-11-17T09:43:12,673 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=CLOSED 2024-11-17T09:43:12,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:43:12,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-17T09:43:12,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 in 160 msec 2024-11-17T09:43:12,677 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, REOPEN/MOVE; state=CLOSED, location=70e37c475432,40671,1731836432285; forceNewPlan=false, retain=true 2024-11-17T09:43:12,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=OPENING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:43:12,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, REOPEN/MOVE because future has completed 2024-11-17T09:43:12,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:43:12,983 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,983 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:43:12,983 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,984 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:43:12,984 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,984 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,985 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,985 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:43:12,986 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b08a5bb92e2e35ac961ce00a4489f23 columnFamilyName A 2024-11-17T09:43:12,986 DEBUG [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:12,987 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(327): Store=0b08a5bb92e2e35ac961ce00a4489f23/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:43:12,987 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,987 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:43:12,988 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b08a5bb92e2e35ac961ce00a4489f23 columnFamilyName B 2024-11-17T09:43:12,988 DEBUG [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:12,988 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(327): Store=0b08a5bb92e2e35ac961ce00a4489f23/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:43:12,988 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,988 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T09:43:12,988 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b08a5bb92e2e35ac961ce00a4489f23 columnFamilyName C 2024-11-17T09:43:12,989 DEBUG [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:12,989 INFO [StoreOpener-0b08a5bb92e2e35ac961ce00a4489f23-1 {}] regionserver.HStore(327): Store=0b08a5bb92e2e35ac961ce00a4489f23/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:43:12,989 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,989 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,990 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,990 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,991 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,991 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,991 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T09:43:12,992 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,993 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 0b08a5bb92e2e35ac961ce00a4489f23; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72199206, jitterRate=0.07585200667381287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T09:43:12,993 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:12,993 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 0b08a5bb92e2e35ac961ce00a4489f23: Running coprocessor pre-open hook at 1731836592984Writing region info on filesystem at 1731836592984Initializing all the Stores at 1731836592984Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836592984Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836592985 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836592985Cleaning up temporary data from old regions at 1731836592991 (+6 ms)Running coprocessor post-open hooks at 1731836592993 (+2 ms)Region opened successfully at 1731836592993 2024-11-17T09:43:12,994 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., pid=147, masterSystemTime=1731836592980 2024-11-17T09:43:12,995 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,995 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:12,995 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=OPEN, openSeqNum=5, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:43:12,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:43:12,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-17T09:43:12,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 in 168 msec 2024-11-17T09:43:13,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-17T09:43:13,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, REOPEN/MOVE in 485 msec 2024-11-17T09:43:13,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T09:43:13,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-17T09:43:13,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-11-17T09:43:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-17T09:43:13,003 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:13,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,005 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b57232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a743e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a7f8ce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,009 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337fa0d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,011 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e136ffd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,012 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66d0bc6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31958f98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eb909c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604782f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b48b46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29aab0c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e27fb03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692722b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516788c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b652b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@669079a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,026 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3228fb63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,028 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34dd109f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,44533,-1 for getting cluster id 2024-11-17T09:43:13,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:43:13,029 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10613bf0-d77c-4efa-912a-b479d3d211a8' 2024-11-17T09:43:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:43:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10613bf0-d77c-4efa-912a-b479d3d211a8" 2024-11-17T09:43:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82c16b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,44533,-1] 2024-11-17T09:43:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:43:13,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:13,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:43:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d895e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:43:13,031 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,032 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,032 DEBUG [Thread-2413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,032 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,032 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,032 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,034 DEBUG [Thread-2400 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,034 DEBUG [Thread-2400 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-17T09:43:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T09:43:13,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:13,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,037 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,038 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,039 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:13,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:13,039 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,039 DEBUG [Thread-2396 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,039 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:43:13,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,041 DEBUG [Thread-2396 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,041 DEBUG [Thread-2411 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,40671,1731836432285, seqNum=-1] 2024-11-17T09:43:13,041 DEBUG [Thread-2411 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:43:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:43:13,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5] 2024-11-17T09:43:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:13,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:43:13,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:13,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:13,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:13,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:13,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:13,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:13,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836653072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836653072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836653072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836653074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836653075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117619fe6ebd55443f28d3d17bd07bdedd0_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836593048/Put/seqid=0 2024-11-17T09:43:13,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742361_1537 (size=12154) 2024-11-17T09:43:13,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T09:43:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836653180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836653180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836653180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836653189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836653189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:13,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:13,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:13,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:13,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T09:43:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836653390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836653390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836653390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836653399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836653399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,497 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:13,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117619fe6ebd55443f28d3d17bd07bdedd0_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117619fe6ebd55443f28d3d17bd07bdedd0_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:13,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/f3bedbbc590d4bd9bf03bb26036ce819, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:13,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:13,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/f3bedbbc590d4bd9bf03bb26036ce819 is 175, key is test_row_0/A:col10/1731836593048/Put/seqid=0 2024-11-17T09:43:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742362_1538 (size=30955) 2024-11-17T09:43:13,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:13,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T09:43:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836653700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836653700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836653700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836653710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836653710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:13,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/f3bedbbc590d4bd9bf03bb26036ce819 2024-11-17T09:43:13,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/3da9721481a4450e80de587c37b47263 is 50, key is test_row_0/B:col10/1731836593048/Put/seqid=0 2024-11-17T09:43:13,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742363_1539 (size=12001) 2024-11-17T09:43:13,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/3da9721481a4450e80de587c37b47263 2024-11-17T09:43:13,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/c98c4ba009034c458d4cbadcea589337 is 50, key is test_row_0/C:col10/1731836593048/Put/seqid=0 2024-11-17T09:43:13,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742364_1540 (size=12001) 2024-11-17T09:43:13,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:13,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:14,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:14,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T09:43:14,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836654210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:14,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836654210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836654210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:14,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836654219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:14,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836654219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:14,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:14,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:14,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:14,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:14,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/c98c4ba009034c458d4cbadcea589337 2024-11-17T09:43:14,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/f3bedbbc590d4bd9bf03bb26036ce819 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/f3bedbbc590d4bd9bf03bb26036ce819 2024-11-17T09:43:14,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/f3bedbbc590d4bd9bf03bb26036ce819, entries=150, sequenceid=16, filesize=30.2 K 2024-11-17T09:43:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/3da9721481a4450e80de587c37b47263 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/3da9721481a4450e80de587c37b47263 2024-11-17T09:43:14,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/3da9721481a4450e80de587c37b47263, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T09:43:14,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/c98c4ba009034c458d4cbadcea589337 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c98c4ba009034c458d4cbadcea589337 2024-11-17T09:43:14,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c98c4ba009034c458d4cbadcea589337, entries=150, sequenceid=16, filesize=11.7 K 2024-11-17T09:43:14,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1325ms, sequenceid=16, compaction requested=false 2024-11-17T09:43:14,373 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T09:43:14,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:14,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-17T09:43:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:14,419 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:43:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:14,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172b74bcfa43644e60af185b49c81c4e5d_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836593073/Put/seqid=0 2024-11-17T09:43:14,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742365_1541 (size=12154) 2024-11-17T09:43:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,434 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172b74bcfa43644e60af185b49c81c4e5d_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172b74bcfa43644e60af185b49c81c4e5d_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/0dcda6d95db24f6080c5b9117e210c60, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:14,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/0dcda6d95db24f6080c5b9117e210c60 is 175, key is test_row_0/A:col10/1731836593073/Put/seqid=0 2024-11-17T09:43:14,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742366_1542 (size=30955) 2024-11-17T09:43:14,439 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/0dcda6d95db24f6080c5b9117e210c60 2024-11-17T09:43:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/b3ee894519f54bba8b5394b3d9f2e341 is 50, key is test_row_0/B:col10/1731836593073/Put/seqid=0 2024-11-17T09:43:14,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742367_1543 (size=12001) 2024-11-17T09:43:14,448 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/b3ee894519f54bba8b5394b3d9f2e341 2024-11-17T09:43:14,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/bc7868ac49d8426da8859551dc24deaf is 50, key is test_row_0/C:col10/1731836593073/Put/seqid=0 2024-11-17T09:43:14,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742368_1544 (size=12001) 2024-11-17T09:43:14,649 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:43:14,858 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/bc7868ac49d8426da8859551dc24deaf 2024-11-17T09:43:14,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/0dcda6d95db24f6080c5b9117e210c60 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/0dcda6d95db24f6080c5b9117e210c60 2024-11-17T09:43:14,866 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/0dcda6d95db24f6080c5b9117e210c60, entries=150, sequenceid=36, filesize=30.2 K 2024-11-17T09:43:14,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/b3ee894519f54bba8b5394b3d9f2e341 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/b3ee894519f54bba8b5394b3d9f2e341 2024-11-17T09:43:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,869 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/b3ee894519f54bba8b5394b3d9f2e341, entries=150, sequenceid=36, filesize=11.7 K 2024-11-17T09:43:14,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/bc7868ac49d8426da8859551dc24deaf as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bc7868ac49d8426da8859551dc24deaf 2024-11-17T09:43:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,874 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bc7868ac49d8426da8859551dc24deaf, entries=150, sequenceid=36, filesize=11.7 K 2024-11-17T09:43:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,875 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 0b08a5bb92e2e35ac961ce00a4489f23 in 455ms, sequenceid=36, compaction requested=false 2024-11-17T09:43:14,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:14,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-17T09:43:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-17T09:43:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-17T09:43:14,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8370 sec 2024-11-17T09:43:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.8460 sec 2024-11-17T09:43:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-17T09:43:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,180 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,181 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-17T09:43:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T09:43:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,183 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,184 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:15,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:43:15,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:15,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:15,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:15,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:15,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:15,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f0feab1c1a2e4a05bcb3dce46b4e3787_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836595262/Put/seqid=0 2024-11-17T09:43:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742369_1545 (size=21918) 2024-11-17T09:43:15,287 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T09:43:15,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f0feab1c1a2e4a05bcb3dce46b4e3787_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f0feab1c1a2e4a05bcb3dce46b4e3787_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:15,292 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/87657c3296f2410aa38acfc117327f04, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:15,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/87657c3296f2410aa38acfc117327f04 is 175, key is test_row_0/A:col10/1731836595262/Put/seqid=0 2024-11-17T09:43:15,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742370_1546 (size=65323) 2024-11-17T09:43:15,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/87657c3296f2410aa38acfc117327f04 2024-11-17T09:43:15,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/bc2a8c69561e4e03a8db242bd58ec258 is 50, key is test_row_0/B:col10/1731836595262/Put/seqid=0 2024-11-17T09:43:15,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836595287, max timestamp=1731836595303Num uniques 859; 2024-11-17T09:43:15,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:43:15,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836595287, max timestamp=1731836595303Num uniques 859; 2024-11-17T09:43:15,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731836595287, max timestamp=1731836595303Num uniques 859; 2024-11-17T09:43:15,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:43:15,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T09:43:15,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742371_1547 (size=12001) 2024-11-17T09:43:15,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/bc2a8c69561e4e03a8db242bd58ec258 2024-11-17T09:43:15,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/ce99af2cfb55458fbd3d1d74a4c9dc59 is 50, key is test_row_0/C:col10/1731836595262/Put/seqid=0 2024-11-17T09:43:15,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742372_1548 (size=12001) 2024-11-17T09:43:15,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/ce99af2cfb55458fbd3d1d74a4c9dc59 2024-11-17T09:43:15,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/87657c3296f2410aa38acfc117327f04 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/87657c3296f2410aa38acfc117327f04 2024-11-17T09:43:15,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/87657c3296f2410aa38acfc117327f04, entries=350, sequenceid=46, filesize=63.8 K 2024-11-17T09:43:15,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/bc2a8c69561e4e03a8db242bd58ec258 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/bc2a8c69561e4e03a8db242bd58ec258 2024-11-17T09:43:15,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/bc2a8c69561e4e03a8db242bd58ec258, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T09:43:15,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/ce99af2cfb55458fbd3d1d74a4c9dc59 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/ce99af2cfb55458fbd3d1d74a4c9dc59 2024-11-17T09:43:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836655324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836655326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/ce99af2cfb55458fbd3d1d74a4c9dc59, entries=150, sequenceid=46, filesize=11.7 K 2024-11-17T09:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0b08a5bb92e2e35ac961ce00a4489f23 in 71ms, sequenceid=46, compaction requested=true 2024-11-17T09:43:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:15,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:15,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:15,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:15,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:43:15,337 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:15,337 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:15,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T09:43:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:15,337 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=140.89 KB heapSize=400.45 KB 2024-11-17T09:43:15,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836595324, max timestamp=1731836595336Num uniques 1068; 2024-11-17T09:43:15,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T09:43:15,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836595324, max timestamp=1731836595336Num uniques 1068; 2024-11-17T09:43:15,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T09:43:15,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731836595324, max timestamp=1731836595336Num uniques 1068; 2024-11-17T09:43:15,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-17T09:43:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:15,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:15,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:15,347 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:15,347 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:15,347 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:15,347 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:15,347 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:15,347 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:15,347 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/f3bedbbc590d4bd9bf03bb26036ce819, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/0dcda6d95db24f6080c5b9117e210c60, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/87657c3296f2410aa38acfc117327f04] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=124.3 K 2024-11-17T09:43:15,347 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/3da9721481a4450e80de587c37b47263, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/b3ee894519f54bba8b5394b3d9f2e341, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/bc2a8c69561e4e03a8db242bd58ec258] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=35.2 K 2024-11-17T09:43:15,347 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:15,347 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/f3bedbbc590d4bd9bf03bb26036ce819, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/0dcda6d95db24f6080c5b9117e210c60, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/87657c3296f2410aa38acfc117327f04] 2024-11-17T09:43:15,348 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da9721481a4450e80de587c37b47263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731836593047 2024-11-17T09:43:15,348 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3bedbbc590d4bd9bf03bb26036ce819, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731836593047 2024-11-17T09:43:15,348 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ee894519f54bba8b5394b3d9f2e341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836593070 2024-11-17T09:43:15,349 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dcda6d95db24f6080c5b9117e210c60, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836593070 2024-11-17T09:43:15,350 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting bc2a8c69561e4e03a8db242bd58ec258, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836595257 2024-11-17T09:43:15,350 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87657c3296f2410aa38acfc117327f04, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836595250 2024-11-17T09:43:15,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709846b4ef1ec4a33acc91293225b2e77_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836595328/Put/seqid=0 2024-11-17T09:43:15,359 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:15,360 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/fce1a221465c49cca95ba76f1eee7ed6 is 50, key is test_row_0/B:col10/1731836595262/Put/seqid=0 2024-11-17T09:43:15,369 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:15,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742373_1549 (size=14594) 2024-11-17T09:43:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:15,378 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111709846b4ef1ec4a33acc91293225b2e77_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709846b4ef1ec4a33acc91293225b2e77_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d947328c84cc408cae393fe69efb01b3, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d947328c84cc408cae393fe69efb01b3 is 175, key is test_row_0/A:col10/1731836595328/Put/seqid=0 2024-11-17T09:43:15,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836595365, max timestamp=1731836595379Num uniques 859; 2024-11-17T09:43:15,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:43:15,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836595365, max timestamp=1731836595379Num uniques 859; 2024-11-17T09:43:15,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:43:15,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731836595365, max timestamp=1731836595379Num uniques 859; 2024-11-17T09:43:15,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T09:43:15,385 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411177e113ada74f24be19e25cf5bb91cf00b_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:15,387 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411177e113ada74f24be19e25cf5bb91cf00b_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:15,387 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177e113ada74f24be19e25cf5bb91cf00b_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:15,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742375_1551 (size=39549) 2024-11-17T09:43:15,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742374_1550 (size=12104) 2024-11-17T09:43:15,417 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d947328c84cc408cae393fe69efb01b3 2024-11-17T09:43:15,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742376_1552 (size=4469) 2024-11-17T09:43:15,422 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#467 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:15,423 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/92689b9b9d3547568422e9f1f9c263b6 is 175, key is test_row_0/A:col10/1731836595262/Put/seqid=0 2024-11-17T09:43:15,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/2ddf543b9de1472d93451eff1313ef9b is 50, key is test_row_0/B:col10/1731836595328/Put/seqid=0 2024-11-17T09:43:15,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742377_1553 (size=31058) 2024-11-17T09:43:15,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742378_1554 (size=12001) 2024-11-17T09:43:15,437 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/2ddf543b9de1472d93451eff1313ef9b 2024-11-17T09:43:15,440 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/92689b9b9d3547568422e9f1f9c263b6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/92689b9b9d3547568422e9f1f9c263b6 2024-11-17T09:43:15,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8831286a237c4520949e3e3e3bb18893 is 50, key is test_row_0/C:col10/1731836595328/Put/seqid=0 2024-11-17T09:43:15,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742379_1555 (size=12001) 2024-11-17T09:43:15,480 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into 92689b9b9d3547568422e9f1f9c263b6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:15,480 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:15,480 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=13, startTime=1731836595336; duration=0sec 2024-11-17T09:43:15,480 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:15,480 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:15,481 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836655439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836655439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,483 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:15,483 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:15,483 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:15,484 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c98c4ba009034c458d4cbadcea589337, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bc7868ac49d8426da8859551dc24deaf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/ce99af2cfb55458fbd3d1d74a4c9dc59] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=35.2 K 2024-11-17T09:43:15,484 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98c4ba009034c458d4cbadcea589337, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731836593047 2024-11-17T09:43:15,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1731836595419, max timestamp=1731836595446Num uniques 1118; 2024-11-17T09:43:15,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51190, new segment data size=7680, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T09:43:15,484 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc7868ac49d8426da8859551dc24deaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731836593070 2024-11-17T09:43:15,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1731836595419, max timestamp=1731836595446Num uniques 1118; 2024-11-17T09:43:15,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51190, new segment data size=7680, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T09:43:15,485 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce99af2cfb55458fbd3d1d74a4c9dc59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836595257 2024-11-17T09:43:15,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1731836595419, max timestamp=1731836595446Num uniques 1118; 2024-11-17T09:43:15,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51190, new segment data size=7680, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T09:43:15,490 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:15,491 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/2c0e31dd802f47ada6fb61b99f264679 is 50, key is test_row_0/C:col10/1731836595262/Put/seqid=0 2024-11-17T09:43:15,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742380_1556 (size=12104) 2024-11-17T09:43:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T09:43:15,505 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/2c0e31dd802f47ada6fb61b99f264679 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/2c0e31dd802f47ada6fb61b99f264679 2024-11-17T09:43:15,510 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into 2c0e31dd802f47ada6fb61b99f264679(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:15,512 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:15,512 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=13, startTime=1731836595337; duration=0sec 2024-11-17T09:43:15,512 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:15,512 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:15,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731836595502, max timestamp=1731836595523Num uniques 1027; 2024-11-17T09:43:15,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-17T09:43:15,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731836595502, max timestamp=1731836595523Num uniques 1027; 2024-11-17T09:43:15,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731836595502, max timestamp=1731836595523Num uniques 1027; 2024-11-17T09:43:15,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-17T09:43:15,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-17T09:43:15,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731836595572, max timestamp=1731836595587Num uniques 1086; 2024-11-17T09:43:15,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731836595572, max timestamp=1731836595587Num uniques 1086; 2024-11-17T09:43:15,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T09:43:15,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T09:43:15,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731836595572, max timestamp=1731836595587Num uniques 1086; 2024-11-17T09:43:15,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T09:43:15,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731836595610, max timestamp=1731836595624Num uniques 1095; 2024-11-17T09:43:15,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-17T09:43:15,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731836595610, max timestamp=1731836595624Num uniques 1095; 2024-11-17T09:43:15,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-17T09:43:15,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731836595615, max timestamp=1731836595624Num uniques 1095; 2024-11-17T09:43:15,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-17T09:43:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836655658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731836595642, max timestamp=1731836595661Num uniques 1054; 2024-11-17T09:43:15,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-17T09:43:15,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731836595642, max timestamp=1731836595661Num uniques 1154; 2024-11-17T09:43:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-17T09:43:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731836595642, max timestamp=1731836595661Num uniques 1154; 2024-11-17T09:43:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-11-17T09:43:15,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836595693, max timestamp=1731836595701Num uniques 1063; 2024-11-17T09:43:15,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-17T09:43:15,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836595693, max timestamp=1731836595701Num uniques 1013; 2024-11-17T09:43:15,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731836595693, max timestamp=1731836595701Num uniques 1013; 2024-11-17T09:43:15,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-17T09:43:15,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-17T09:43:15,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731836595716, max timestamp=1731836595730Num uniques 1072; 2024-11-17T09:43:15,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731836595716, max timestamp=1731836595730Num uniques 1072; 2024-11-17T09:43:15,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-17T09:43:15,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-17T09:43:15,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731836595716, max timestamp=1731836595730Num uniques 1072; 2024-11-17T09:43:15,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-17T09:43:15,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836655757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836655760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731836595755, max timestamp=1731836595761Num uniques 1131; 2024-11-17T09:43:15,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-17T09:43:15,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836655761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731836595755, max timestamp=1731836595761Num uniques 1131; 2024-11-17T09:43:15,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731836595755, max timestamp=1731836595761Num uniques 1131; 2024-11-17T09:43:15,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-17T09:43:15,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=10576, suffix heap size=145592,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=231 2024-11-17T09:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731836595796, max timestamp=1731836595807Num uniques 1090; 2024-11-17T09:43:15,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-17T09:43:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731836595796, max timestamp=1731836595807Num uniques 1090; 2024-11-17T09:43:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-17T09:43:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731836595796, max timestamp=1731836595807Num uniques 1090; 2024-11-17T09:43:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-17T09:43:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T09:43:15,820 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/fce1a221465c49cca95ba76f1eee7ed6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fce1a221465c49cca95ba76f1eee7ed6 2024-11-17T09:43:15,824 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into fce1a221465c49cca95ba76f1eee7ed6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:15,824 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:15,824 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=13, startTime=1731836595336; duration=0sec 2024-11-17T09:43:15,824 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:15,824 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:15,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731836595838, max timestamp=1731836595848Num uniques 1099; 2024-11-17T09:43:15,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-17T09:43:15,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731836595838, max timestamp=1731836595848Num uniques 1099; 2024-11-17T09:43:15,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-17T09:43:15,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731836595838, max timestamp=1731836595848Num uniques 1099; 2024-11-17T09:43:15,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-17T09:43:15,865 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8831286a237c4520949e3e3e3bb18893 2024-11-17T09:43:15,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d947328c84cc408cae393fe69efb01b3 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d947328c84cc408cae393fe69efb01b3 2024-11-17T09:43:15,883 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d947328c84cc408cae393fe69efb01b3, entries=200, sequenceid=84, filesize=38.6 K 2024-11-17T09:43:15,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/2ddf543b9de1472d93451eff1313ef9b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2ddf543b9de1472d93451eff1313ef9b 2024-11-17T09:43:15,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836655881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836655883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731836595877, max timestamp=1731836595885Num uniques 1108; 2024-11-17T09:43:15,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-17T09:43:15,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731836595877, max timestamp=1731836595885Num uniques 1108; 2024-11-17T09:43:15,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-17T09:43:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731836595877, max timestamp=1731836595885Num uniques 1108; 2024-11-17T09:43:15,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=6870, suffix heap size=143200,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=150 2024-11-17T09:43:15,894 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2ddf543b9de1472d93451eff1313ef9b, entries=150, sequenceid=84, filesize=11.7 K 2024-11-17T09:43:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8831286a237c4520949e3e3e3bb18893 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8831286a237c4520949e3e3e3bb18893 2024-11-17T09:43:15,898 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8831286a237c4520949e3e3e3bb18893, entries=150, sequenceid=84, filesize=11.7 K 2024-11-17T09:43:15,899 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=57.79 KB/59180 for 0b08a5bb92e2e35ac961ce00a4489f23 in 562ms, sequenceid=84, compaction requested=false 2024-11-17T09:43:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-17T09:43:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-17T09:43:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:15,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=71.21 KB heapSize=195.20 KB 2024-11-17T09:43:15,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:15,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:15,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T09:43:15,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 716 msec 2024-11-17T09:43:15,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 721 msec 2024-11-17T09:43:15,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171a86ed754596407085a7fc3cab67926c_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836595894/Put/seqid=0 2024-11-17T09:43:15,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742381_1557 (size=12454) 2024-11-17T09:43:15,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836655964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836655964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836655964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836656000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836656000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836656079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836656079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836656080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836656210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836656210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836656290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836656290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836656290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:16,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171a86ed754596407085a7fc3cab67926c_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171a86ed754596407085a7fc3cab67926c_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:16,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/553ea4dee8484e62b89b6df9f83adbde, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:16,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/553ea4dee8484e62b89b6df9f83adbde is 175, key is test_row_0/A:col10/1731836595894/Put/seqid=0 2024-11-17T09:43:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T09:43:16,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742382_1558 (size=31255) 2024-11-17T09:43:16,320 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:16,320 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-17T09:43:16,321 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T09:43:16,322 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:16,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T09:43:16,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836656520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836656520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836656600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836656600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836656600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:16,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T09:43:16,720 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/553ea4dee8484e62b89b6df9f83adbde 2024-11-17T09:43:16,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/58766491abd64cdabfa2d56754dba062 is 50, key is test_row_0/B:col10/1731836595894/Put/seqid=0 2024-11-17T09:43:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742383_1559 (size=12301) 2024-11-17T09:43:16,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:16,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:16,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:16,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T09:43:17,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836657029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836657029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:17,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836657109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:17,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836657109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:17,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836657120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:17,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/58766491abd64cdabfa2d56754dba062 2024-11-17T09:43:17,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/cc29bab920644dd1b7ab54d292cfa3fe is 50, key is test_row_0/C:col10/1731836595894/Put/seqid=0 2024-11-17T09:43:17,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742384_1560 (size=12301) 2024-11-17T09:43:17,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:17,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T09:43:17,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.01 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/cc29bab920644dd1b7ab54d292cfa3fe 2024-11-17T09:43:17,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:17,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/553ea4dee8484e62b89b6df9f83adbde as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/553ea4dee8484e62b89b6df9f83adbde 2024-11-17T09:43:17,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/553ea4dee8484e62b89b6df9f83adbde, entries=150, sequenceid=303, filesize=30.5 K 2024-11-17T09:43:17,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/58766491abd64cdabfa2d56754dba062 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/58766491abd64cdabfa2d56754dba062 2024-11-17T09:43:17,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/58766491abd64cdabfa2d56754dba062, entries=150, sequenceid=303, filesize=12.0 K 2024-11-17T09:43:17,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/cc29bab920644dd1b7ab54d292cfa3fe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/cc29bab920644dd1b7ab54d292cfa3fe 2024-11-17T09:43:17,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/cc29bab920644dd1b7ab54d292cfa3fe, entries=150, sequenceid=303, filesize=12.0 K 2024-11-17T09:43:17,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.21 KB/72920, heapSize ~195.16 KB/199840, currentSize=114.05 KB/116790 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1658ms, sequenceid=303, compaction requested=true 2024-11-17T09:43:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:17,560 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:17,560 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:17,561 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101862 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:17,561 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:17,561 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:17,561 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,561 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:17,561 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/92689b9b9d3547568422e9f1f9c263b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d947328c84cc408cae393fe69efb01b3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/553ea4dee8484e62b89b6df9f83adbde] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=99.5 K 2024-11-17T09:43:17,561 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,561 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,561 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fce1a221465c49cca95ba76f1eee7ed6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2ddf543b9de1472d93451eff1313ef9b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/58766491abd64cdabfa2d56754dba062] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=35.6 K 2024-11-17T09:43:17,561 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/92689b9b9d3547568422e9f1f9c263b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d947328c84cc408cae393fe69efb01b3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/553ea4dee8484e62b89b6df9f83adbde] 2024-11-17T09:43:17,561 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92689b9b9d3547568422e9f1f9c263b6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836595257 2024-11-17T09:43:17,561 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting fce1a221465c49cca95ba76f1eee7ed6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836595257 2024-11-17T09:43:17,562 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ddf543b9de1472d93451eff1313ef9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836595324 2024-11-17T09:43:17,562 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d947328c84cc408cae393fe69efb01b3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836595324 2024-11-17T09:43:17,562 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 58766491abd64cdabfa2d56754dba062, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731836595894 2024-11-17T09:43:17,562 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553ea4dee8484e62b89b6df9f83adbde, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731836595894 2024-11-17T09:43:17,568 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:17,570 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:17,570 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/762c0fc6947f4541921ee9834e13c614 is 50, key is test_row_0/B:col10/1731836595894/Put/seqid=0 2024-11-17T09:43:17,571 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117c07a6480a0904149943f5654cfca83a6_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:17,574 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117c07a6480a0904149943f5654cfca83a6_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:17,574 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c07a6480a0904149943f5654cfca83a6_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:17,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742385_1561 (size=12507) 2024-11-17T09:43:17,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742386_1562 (size=4469) 2024-11-17T09:43:17,592 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#475 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:17,592 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/2afebe014a2a4d5e9693ddb8cd33308d is 175, key is test_row_0/A:col10/1731836595894/Put/seqid=0 2024-11-17T09:43:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742387_1563 (size=31461) 2024-11-17T09:43:17,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T09:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,696 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:17,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b31ee87495d54f66a535c856d2bca761_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836595945/Put/seqid=0 2024-11-17T09:43:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742388_1564 (size=12454) 2024-11-17T09:43:17,983 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/762c0fc6947f4541921ee9834e13c614 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/762c0fc6947f4541921ee9834e13c614 2024-11-17T09:43:17,987 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into 762c0fc6947f4541921ee9834e13c614(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:17,987 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:17,987 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=13, startTime=1731836597560; duration=0sec 2024-11-17T09:43:17,987 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:17,987 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:17,987 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:17,988 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:17,988 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:17,988 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:17,988 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/2c0e31dd802f47ada6fb61b99f264679, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8831286a237c4520949e3e3e3bb18893, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/cc29bab920644dd1b7ab54d292cfa3fe] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=35.6 K 2024-11-17T09:43:17,988 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c0e31dd802f47ada6fb61b99f264679, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731836595257 2024-11-17T09:43:17,989 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8831286a237c4520949e3e3e3bb18893, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731836595324 2024-11-17T09:43:17,989 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting cc29bab920644dd1b7ab54d292cfa3fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731836595894 2024-11-17T09:43:17,994 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:17,995 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/1a87b0d84eff4fa7989f9197afe803fc is 50, key is test_row_0/C:col10/1731836595894/Put/seqid=0 2024-11-17T09:43:17,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742389_1565 (size=12507) 2024-11-17T09:43:18,003 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/1a87b0d84eff4fa7989f9197afe803fc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1a87b0d84eff4fa7989f9197afe803fc 2024-11-17T09:43:18,003 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/2afebe014a2a4d5e9693ddb8cd33308d as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2afebe014a2a4d5e9693ddb8cd33308d 2024-11-17T09:43:18,007 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into 2afebe014a2a4d5e9693ddb8cd33308d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:18,007 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:18,007 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=13, startTime=1731836597560; duration=0sec 2024-11-17T09:43:18,007 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into 1a87b0d84eff4fa7989f9197afe803fc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:18,007 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:18,007 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:18,007 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=13, startTime=1731836597560; duration=0sec 2024-11-17T09:43:18,007 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:18,007 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:18,007 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:18,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:18,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836658061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836658063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:18,111 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b31ee87495d54f66a535c856d2bca761_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b31ee87495d54f66a535c856d2bca761_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:18,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3902844176384bca8b4814158469466f, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3902844176384bca8b4814158469466f is 175, key is test_row_0/A:col10/1731836595945/Put/seqid=0 2024-11-17T09:43:18,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742390_1566 (size=31255) 2024-11-17T09:43:18,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836658120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,122 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836658120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,124 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836658140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,142 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836658170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836658170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836658380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836658380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T09:43:18,516 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3902844176384bca8b4814158469466f 2024-11-17T09:43:18,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c66a79ed066d4288835360f93a2a6853 is 50, key is test_row_0/B:col10/1731836595945/Put/seqid=0 2024-11-17T09:43:18,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742391_1567 (size=12301) 2024-11-17T09:43:18,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836658689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:18,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836658689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:18,926 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c66a79ed066d4288835360f93a2a6853 2024-11-17T09:43:18,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/3fb23d293c0f4b50897ecafc297bc0ab is 50, key is test_row_0/C:col10/1731836595945/Put/seqid=0 2024-11-17T09:43:18,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742392_1568 (size=12301) 2024-11-17T09:43:19,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836659200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:19,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:19,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836659200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:19,338 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/3fb23d293c0f4b50897ecafc297bc0ab 2024-11-17T09:43:19,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3902844176384bca8b4814158469466f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3902844176384bca8b4814158469466f 2024-11-17T09:43:19,345 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3902844176384bca8b4814158469466f, entries=150, sequenceid=323, filesize=30.5 K 2024-11-17T09:43:19,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c66a79ed066d4288835360f93a2a6853 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c66a79ed066d4288835360f93a2a6853 2024-11-17T09:43:19,348 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c66a79ed066d4288835360f93a2a6853, entries=150, sequenceid=323, filesize=12.0 K 2024-11-17T09:43:19,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/3fb23d293c0f4b50897ecafc297bc0ab as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/3fb23d293c0f4b50897ecafc297bc0ab 2024-11-17T09:43:19,352 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/3fb23d293c0f4b50897ecafc297bc0ab, entries=150, sequenceid=323, filesize=12.0 K 2024-11-17T09:43:19,352 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1656ms, sequenceid=323, compaction requested=false 2024-11-17T09:43:19,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:19,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-17T09:43:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-17T09:43:19,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-17T09:43:19,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0320 sec 2024-11-17T09:43:19,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0360 sec 2024-11-17T09:43:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:20,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T09:43:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:20,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:20,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e448614cf0fa413692fa914829ddcf1f_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836598060/Put/seqid=0 2024-11-17T09:43:20,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742393_1569 (size=14994) 2024-11-17T09:43:20,158 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:20,161 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e448614cf0fa413692fa914829ddcf1f_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e448614cf0fa413692fa914829ddcf1f_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:20,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3068cd33967b4cc4bd5682c4574a776a, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:20,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3068cd33967b4cc4bd5682c4574a776a is 175, key is test_row_0/A:col10/1731836598060/Put/seqid=0 2024-11-17T09:43:20,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742394_1570 (size=39949) 2024-11-17T09:43:20,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836660173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836660177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836660178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836660210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,212 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836660220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,221 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836660290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836660290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836660290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T09:43:20,490 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:20,490 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T09:43:20,492 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T09:43:20,492 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:20,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836660499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836660500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836660500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3068cd33967b4cc4bd5682c4574a776a 2024-11-17T09:43:20,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/4375e133ca624097a5fdac691dad5b86 is 50, key is test_row_0/B:col10/1731836598060/Put/seqid=0 2024-11-17T09:43:20,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742395_1571 (size=12301) 2024-11-17T09:43:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T09:43:20,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T09:43:20,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:20,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:20,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:20,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T09:43:20,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:20,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:20,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:20,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T09:43:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836660810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836660810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836660810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:20,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T09:43:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:20,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:20,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/4375e133ca624097a5fdac691dad5b86 2024-11-17T09:43:20,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8c9de2c8516349c79b652250bf3a4b79 is 50, key is test_row_0/C:col10/1731836598060/Put/seqid=0 2024-11-17T09:43:20,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742396_1572 (size=12301) 2024-11-17T09:43:21,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T09:43:21,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:21,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:21,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T09:43:21,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T09:43:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:21,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836661320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:21,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836661320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:21,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836661330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:21,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8c9de2c8516349c79b652250bf3a4b79 2024-11-17T09:43:21,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3068cd33967b4cc4bd5682c4574a776a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3068cd33967b4cc4bd5682c4574a776a 2024-11-17T09:43:21,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3068cd33967b4cc4bd5682c4574a776a, entries=200, sequenceid=339, filesize=39.0 K 2024-11-17T09:43:21,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/4375e133ca624097a5fdac691dad5b86 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/4375e133ca624097a5fdac691dad5b86 2024-11-17T09:43:21,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/4375e133ca624097a5fdac691dad5b86, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T09:43:21,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8c9de2c8516349c79b652250bf3a4b79 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8c9de2c8516349c79b652250bf3a4b79 2024-11-17T09:43:21,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8c9de2c8516349c79b652250bf3a4b79, entries=150, sequenceid=339, filesize=12.0 K 2024-11-17T09:43:21,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1272ms, sequenceid=339, compaction requested=true 2024-11-17T09:43:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:21,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:21,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:21,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:21,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:21,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:21,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T09:43:21,406 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:21,406 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:21,407 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:21,407 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:21,407 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,407 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2afebe014a2a4d5e9693ddb8cd33308d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3902844176384bca8b4814158469466f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3068cd33967b4cc4bd5682c4574a776a] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=100.3 K 2024-11-17T09:43:21,407 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,407 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2afebe014a2a4d5e9693ddb8cd33308d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3902844176384bca8b4814158469466f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3068cd33967b4cc4bd5682c4574a776a] 2024-11-17T09:43:21,407 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:21,407 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:21,408 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,408 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/762c0fc6947f4541921ee9834e13c614, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c66a79ed066d4288835360f93a2a6853, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/4375e133ca624097a5fdac691dad5b86] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.2 K 2024-11-17T09:43:21,408 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2afebe014a2a4d5e9693ddb8cd33308d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731836595894 2024-11-17T09:43:21,408 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 762c0fc6947f4541921ee9834e13c614, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731836595894 2024-11-17T09:43:21,409 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c66a79ed066d4288835360f93a2a6853, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731836595912 2024-11-17T09:43:21,409 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3902844176384bca8b4814158469466f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731836595912 2024-11-17T09:43:21,410 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 4375e133ca624097a5fdac691dad5b86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836598060 2024-11-17T09:43:21,411 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3068cd33967b4cc4bd5682c4574a776a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836598057 2024-11-17T09:43:21,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T09:43:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,412 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T09:43:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:21,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:21,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:21,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:21,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:21,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111757f61c9a942e43be83d86ebdb4427537_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836600172/Put/seqid=0 2024-11-17T09:43:21,447 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:21,447 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/02ab228b4de84afab63ec05aa6c347ad is 50, key is test_row_0/B:col10/1731836598060/Put/seqid=0 2024-11-17T09:43:21,448 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:21,457 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411176021afc57fb540b2ae6251af346ff4c4_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:21,459 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411176021afc57fb540b2ae6251af346ff4c4_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:21,459 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176021afc57fb540b2ae6251af346ff4c4_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:21,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742397_1573 (size=12454) 2024-11-17T09:43:21,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,477 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111757f61c9a942e43be83d86ebdb4427537_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757f61c9a942e43be83d86ebdb4427537_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/aa12d45a3cfc43edb272e2320536be59, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/aa12d45a3cfc43edb272e2320536be59 is 175, key is test_row_0/A:col10/1731836600172/Put/seqid=0 2024-11-17T09:43:21,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742398_1574 (size=12609) 2024-11-17T09:43:21,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742399_1575 (size=4469) 2024-11-17T09:43:21,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742400_1576 (size=31255) 2024-11-17T09:43:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T09:43:21,890 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#485 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:21,891 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/02919d9cd0bf4528bd8fd1d0b49b3ff4 is 175, key is test_row_0/A:col10/1731836598060/Put/seqid=0 2024-11-17T09:43:21,893 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/02ab228b4de84afab63ec05aa6c347ad as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/02ab228b4de84afab63ec05aa6c347ad 2024-11-17T09:43:21,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742401_1577 (size=31563) 2024-11-17T09:43:21,894 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/aa12d45a3cfc43edb272e2320536be59 2024-11-17T09:43:21,898 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into 02ab228b4de84afab63ec05aa6c347ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:21,898 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:21,898 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=13, startTime=1731836601406; duration=0sec 2024-11-17T09:43:21,898 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:21,898 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:21,898 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:21,899 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/02919d9cd0bf4528bd8fd1d0b49b3ff4 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/02919d9cd0bf4528bd8fd1d0b49b3ff4 2024-11-17T09:43:21,900 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:21,900 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:21,900 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,900 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1a87b0d84eff4fa7989f9197afe803fc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/3fb23d293c0f4b50897ecafc297bc0ab, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8c9de2c8516349c79b652250bf3a4b79] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.2 K 2024-11-17T09:43:21,900 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a87b0d84eff4fa7989f9197afe803fc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731836595894 2024-11-17T09:43:21,901 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb23d293c0f4b50897ecafc297bc0ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731836595912 2024-11-17T09:43:21,901 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9de2c8516349c79b652250bf3a4b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836598060 2024-11-17T09:43:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c4f73eb9f0a443e898866e7f8cee7b68 is 50, key is test_row_0/B:col10/1731836600172/Put/seqid=0 2024-11-17T09:43:21,907 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into 02919d9cd0bf4528bd8fd1d0b49b3ff4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:21,907 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:21,907 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=13, startTime=1731836601406; duration=0sec 2024-11-17T09:43:21,907 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:21,907 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:21,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742402_1578 (size=12301) 2024-11-17T09:43:21,913 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:21,914 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/a6792c8825e84fd9a6af8acaab886bb2 is 50, key is test_row_0/C:col10/1731836598060/Put/seqid=0 2024-11-17T09:43:21,914 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c4f73eb9f0a443e898866e7f8cee7b68 2024-11-17T09:43:21,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742403_1579 (size=12609) 2024-11-17T09:43:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8afd3cdfab704c5e838c4e0f4f887d89 is 50, key is test_row_0/C:col10/1731836600172/Put/seqid=0 2024-11-17T09:43:21,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742404_1580 (size=12301) 2024-11-17T09:43:21,932 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8afd3cdfab704c5e838c4e0f4f887d89 2024-11-17T09:43:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/aa12d45a3cfc43edb272e2320536be59 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/aa12d45a3cfc43edb272e2320536be59 2024-11-17T09:43:21,938 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/aa12d45a3cfc43edb272e2320536be59, entries=150, sequenceid=358, filesize=30.5 K 2024-11-17T09:43:21,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c4f73eb9f0a443e898866e7f8cee7b68 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c4f73eb9f0a443e898866e7f8cee7b68 2024-11-17T09:43:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,942 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c4f73eb9f0a443e898866e7f8cee7b68, entries=150, sequenceid=358, filesize=12.0 K 2024-11-17T09:43:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8afd3cdfab704c5e838c4e0f4f887d89 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8afd3cdfab704c5e838c4e0f4f887d89 2024-11-17T09:43:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,947 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8afd3cdfab704c5e838c4e0f4f887d89, entries=150, sequenceid=358, filesize=12.0 K 2024-11-17T09:43:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,947 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 0b08a5bb92e2e35ac961ce00a4489f23 in 535ms, sequenceid=358, compaction requested=false 2024-11-17T09:43:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T09:43:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T09:43:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T09:43:21,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4580 sec 2024-11-17T09:43:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4630 sec 2024-11-17T09:43:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:22,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T09:43:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb809246c4464adca527c14abe88e8b2_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836602298/Put/seqid=0 2024-11-17T09:43:22,324 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/a6792c8825e84fd9a6af8acaab886bb2 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a6792c8825e84fd9a6af8acaab886bb2 2024-11-17T09:43:22,328 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into a6792c8825e84fd9a6af8acaab886bb2(size=12.3 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:22,328 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:22,329 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=13, startTime=1731836601406; duration=0sec 2024-11-17T09:43:22,329 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:22,329 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:22,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836602344, max timestamp=1731836602349Num uniques 859; 2024-11-17T09:43:22,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:22,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836602344, max timestamp=1731836602349Num uniques 859; 2024-11-17T09:43:22,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:22,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836602344, max timestamp=1731836602349Num uniques 859; 2024-11-17T09:43:22,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:22,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742405_1581 (size=17534) 2024-11-17T09:43:22,360 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:22,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb809246c4464adca527c14abe88e8b2_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb809246c4464adca527c14abe88e8b2_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:22,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/774c9b4a72b74dbfbc33772c094d45c9, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:22,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/774c9b4a72b74dbfbc33772c094d45c9 is 175, key is test_row_0/A:col10/1731836602298/Put/seqid=0 2024-11-17T09:43:22,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742406_1582 (size=48635) 2024-11-17T09:43:22,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836602371, max timestamp=1731836602379Num uniques 1018; 2024-11-17T09:43:22,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:43:22,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836602371, max timestamp=1731836602379Num uniques 1018; 2024-11-17T09:43:22,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:43:22,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731836602371, max timestamp=1731836602379Num uniques 1018; 2024-11-17T09:43:22,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T09:43:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836662404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836662405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836662407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836662408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836662409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836662520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836662520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836662520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836662520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836662520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T09:43:22,650 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:22,650 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T09:43:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T09:43:22,652 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:22,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:22,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:22,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836662730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836662730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836662730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836662730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836662731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:22,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T09:43:22,772 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/774c9b4a72b74dbfbc33772c094d45c9 2024-11-17T09:43:22,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c1117078d51d4efe9e739a30c3e396fe is 50, key is test_row_0/B:col10/1731836602298/Put/seqid=0 2024-11-17T09:43:22,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742407_1583 (size=9857) 2024-11-17T09:43:22,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c1117078d51d4efe9e739a30c3e396fe 2024-11-17T09:43:22,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/c129ff1c677545b4bbbc6ec350242c44 is 50, key is test_row_0/C:col10/1731836602298/Put/seqid=0 2024-11-17T09:43:22,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742408_1584 (size=9857) 2024-11-17T09:43:22,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T09:43:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:22,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:22,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T09:43:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:22,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T09:43:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836663040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836663040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836663040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836663040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836663041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T09:43:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:23,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:23,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/c129ff1c677545b4bbbc6ec350242c44 2024-11-17T09:43:23,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/774c9b4a72b74dbfbc33772c094d45c9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/774c9b4a72b74dbfbc33772c094d45c9 2024-11-17T09:43:23,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/774c9b4a72b74dbfbc33772c094d45c9, entries=250, sequenceid=370, filesize=47.5 K 2024-11-17T09:43:23,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/c1117078d51d4efe9e739a30c3e396fe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c1117078d51d4efe9e739a30c3e396fe 2024-11-17T09:43:23,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c1117078d51d4efe9e739a30c3e396fe, entries=100, sequenceid=370, filesize=9.6 K 2024-11-17T09:43:23,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/c129ff1c677545b4bbbc6ec350242c44 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c129ff1c677545b4bbbc6ec350242c44 2024-11-17T09:43:23,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c129ff1c677545b4bbbc6ec350242c44, entries=100, sequenceid=370, filesize=9.6 K 2024-11-17T09:43:23,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0b08a5bb92e2e35ac961ce00a4489f23 in 902ms, sequenceid=370, compaction requested=true 2024-11-17T09:43:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:23,209 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:23,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:23,209 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:23,210 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:23,210 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:23,210 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:23,210 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:23,210 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:23,210 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:23,210 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/02919d9cd0bf4528bd8fd1d0b49b3ff4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/aa12d45a3cfc43edb272e2320536be59, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/774c9b4a72b74dbfbc33772c094d45c9] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=108.8 K 2024-11-17T09:43:23,210 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/02ab228b4de84afab63ec05aa6c347ad, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c4f73eb9f0a443e898866e7f8cee7b68, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c1117078d51d4efe9e739a30c3e396fe] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=34.0 K 2024-11-17T09:43:23,210 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:23,210 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/02919d9cd0bf4528bd8fd1d0b49b3ff4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/aa12d45a3cfc43edb272e2320536be59, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/774c9b4a72b74dbfbc33772c094d45c9] 2024-11-17T09:43:23,211 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02919d9cd0bf4528bd8fd1d0b49b3ff4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836598060 2024-11-17T09:43:23,211 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ab228b4de84afab63ec05aa6c347ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836598060 2024-11-17T09:43:23,211 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f73eb9f0a443e898866e7f8cee7b68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731836600172 2024-11-17T09:43:23,211 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa12d45a3cfc43edb272e2320536be59, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731836600172 2024-11-17T09:43:23,211 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c1117078d51d4efe9e739a30c3e396fe, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731836602298 2024-11-17T09:43:23,211 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774c9b4a72b74dbfbc33772c094d45c9, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731836602279 2024-11-17T09:43:23,216 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:23,219 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:23,219 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/63fb4f51a9ab4c42a5420fd45e5bcd5f is 50, key is test_row_0/B:col10/1731836602298/Put/seqid=0 2024-11-17T09:43:23,221 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411174bb80e670b8d4cba9eb589415b4e0587_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:23,223 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411174bb80e670b8d4cba9eb589415b4e0587_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:23,223 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174bb80e670b8d4cba9eb589415b4e0587_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:23,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742409_1585 (size=12711) 2024-11-17T09:43:23,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742410_1586 (size=4469) 2024-11-17T09:43:23,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T09:43:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:23,264 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=134.18 KB heapSize=384.14 KB 2024-11-17T09:43:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:23,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117611009a1dc664161b0402511f91d285e_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836602406/Put/seqid=0 2024-11-17T09:43:23,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742411_1587 (size=12454) 2024-11-17T09:43:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T09:43:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:23,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:23,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836663554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836663555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836663560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836663561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836663561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,629 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:23,630 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/62393bf781af475097c7c933ba9f726b is 175, key is test_row_0/A:col10/1731836602298/Put/seqid=0 2024-11-17T09:43:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742412_1588 (size=31772) 2024-11-17T09:43:23,637 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/62393bf781af475097c7c933ba9f726b as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/62393bf781af475097c7c933ba9f726b 2024-11-17T09:43:23,637 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/63fb4f51a9ab4c42a5420fd45e5bcd5f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/63fb4f51a9ab4c42a5420fd45e5bcd5f 2024-11-17T09:43:23,642 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into 63fb4f51a9ab4c42a5420fd45e5bcd5f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:23,642 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:23,642 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=13, startTime=1731836603209; duration=0sec 2024-11-17T09:43:23,642 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:23,642 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:23,642 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:23,643 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into 62393bf781af475097c7c933ba9f726b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:23,643 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:23,643 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=13, startTime=1731836603209; duration=0sec 2024-11-17T09:43:23,643 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:23,643 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:23,644 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:23,644 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:23,644 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:23,644 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a6792c8825e84fd9a6af8acaab886bb2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8afd3cdfab704c5e838c4e0f4f887d89, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c129ff1c677545b4bbbc6ec350242c44] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=34.0 K 2024-11-17T09:43:23,645 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a6792c8825e84fd9a6af8acaab886bb2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731836598060 2024-11-17T09:43:23,645 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8afd3cdfab704c5e838c4e0f4f887d89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731836600172 2024-11-17T09:43:23,646 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting c129ff1c677545b4bbbc6ec350242c44, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731836602298 2024-11-17T09:43:23,652 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:23,653 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/206fafee81eb44e48736af1df8ab6eed is 50, key is test_row_0/C:col10/1731836602298/Put/seqid=0 2024-11-17T09:43:23,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742413_1589 (size=12711) 2024-11-17T09:43:23,663 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/206fafee81eb44e48736af1df8ab6eed as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/206fafee81eb44e48736af1df8ab6eed 2024-11-17T09:43:23,667 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into 206fafee81eb44e48736af1df8ab6eed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:23,667 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:23,667 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=13, startTime=1731836603209; duration=0sec 2024-11-17T09:43:23,668 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:23,668 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836663670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836663670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836663671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836663671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,677 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117611009a1dc664161b0402511f91d285e_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117611009a1dc664161b0402511f91d285e_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/8592ddaafb4d42b2ad65ab27a4f2e719, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/8592ddaafb4d42b2ad65ab27a4f2e719 is 175, key is test_row_0/A:col10/1731836602406/Put/seqid=0 2024-11-17T09:43:23,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742414_1590 (size=31255) 2024-11-17T09:43:23,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836663680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T09:43:23,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836663879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836663879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836663880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836663880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:23,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:23,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836663890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:23,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:23,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,082 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/8592ddaafb4d42b2ad65ab27a4f2e719 2024-11-17T09:43:24,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/fe6e7dd017ec4b4088930c0cfc9ba9fe is 50, key is test_row_0/B:col10/1731836602406/Put/seqid=0 2024-11-17T09:43:24,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742415_1591 (size=12301) 2024-11-17T09:43:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836664190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836664190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836664190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836664190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836664200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,492 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/fe6e7dd017ec4b4088930c0cfc9ba9fe 2024-11-17T09:43:24,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/07453b45c1c94722b934f59161f2f205 is 50, key is test_row_0/C:col10/1731836602406/Put/seqid=0 2024-11-17T09:43:24,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742416_1592 (size=12301) 2024-11-17T09:43:24,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836664699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836664699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836664710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836664710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836664710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T09:43:24,903 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/07453b45c1c94722b934f59161f2f205 2024-11-17T09:43:24,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/8592ddaafb4d42b2ad65ab27a4f2e719 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/8592ddaafb4d42b2ad65ab27a4f2e719 2024-11-17T09:43:24,910 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/8592ddaafb4d42b2ad65ab27a4f2e719, entries=150, sequenceid=425, filesize=30.5 K 2024-11-17T09:43:24,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/fe6e7dd017ec4b4088930c0cfc9ba9fe as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fe6e7dd017ec4b4088930c0cfc9ba9fe 2024-11-17T09:43:24,914 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fe6e7dd017ec4b4088930c0cfc9ba9fe, entries=150, sequenceid=425, filesize=12.0 K 2024-11-17T09:43:24,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/07453b45c1c94722b934f59161f2f205 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/07453b45c1c94722b934f59161f2f205 2024-11-17T09:43:24,918 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/07453b45c1c94722b934f59161f2f205, entries=150, sequenceid=425, filesize=12.0 K 2024-11-17T09:43:24,918 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.09 KB/393312, currentSize=53.67 KB/54960 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1654ms, sequenceid=425, compaction requested=false 2024-11-17T09:43:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T09:43:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T09:43:24,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T09:43:24,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2670 sec 2024-11-17T09:43:24,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2720 sec 2024-11-17T09:43:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:25,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T09:43:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:25,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170dfddfb0505f454abf5c3d4a688da57c_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836605712/Put/seqid=0 2024-11-17T09:43:25,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742417_1593 (size=12454) 2024-11-17T09:43:25,725 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:25,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170dfddfb0505f454abf5c3d4a688da57c_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dfddfb0505f454abf5c3d4a688da57c_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:25,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/2381d66321974c68bd8ac3aa0ccf7af9, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:25,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/2381d66321974c68bd8ac3aa0ccf7af9 is 175, key is test_row_0/A:col10/1731836605712/Put/seqid=0 2024-11-17T09:43:25,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742418_1594 (size=31255) 2024-11-17T09:43:25,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836665739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836665746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836665746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836665747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836665747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836665850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836665860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836665860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836665861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836665861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836666060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836666069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836666070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836666070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836666070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,135 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=441, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/2381d66321974c68bd8ac3aa0ccf7af9 2024-11-17T09:43:26,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/ca25d32e606a45aca8824a07e3c1d6db is 50, key is test_row_0/B:col10/1731836605712/Put/seqid=0 2024-11-17T09:43:26,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742419_1595 (size=12301) 2024-11-17T09:43:26,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836666370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836666379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836666379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836666380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836666380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/ca25d32e606a45aca8824a07e3c1d6db 2024-11-17T09:43:26,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/be2883c152f64d6fb9ea5400d84ec0e6 is 50, key is test_row_0/C:col10/1731836605712/Put/seqid=0 2024-11-17T09:43:26,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742420_1596 (size=12301) 2024-11-17T09:43:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T09:43:26,810 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:26,810 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T09:43:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T09:43:26,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:26,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:26,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:26,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836666889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836666890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836666890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836666890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836666890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:26,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:26,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T09:43:26,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/be2883c152f64d6fb9ea5400d84ec0e6 2024-11-17T09:43:26,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/2381d66321974c68bd8ac3aa0ccf7af9 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2381d66321974c68bd8ac3aa0ccf7af9 2024-11-17T09:43:26,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2381d66321974c68bd8ac3aa0ccf7af9, entries=150, sequenceid=441, filesize=30.5 K 2024-11-17T09:43:26,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/ca25d32e606a45aca8824a07e3c1d6db as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/ca25d32e606a45aca8824a07e3c1d6db 2024-11-17T09:43:26,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T09:43:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:26,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:26,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/ca25d32e606a45aca8824a07e3c1d6db, entries=150, sequenceid=441, filesize=12.0 K 2024-11-17T09:43:26,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/be2883c152f64d6fb9ea5400d84ec0e6 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/be2883c152f64d6fb9ea5400d84ec0e6 2024-11-17T09:43:26,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/be2883c152f64d6fb9ea5400d84ec0e6, entries=150, sequenceid=441, filesize=12.0 K 2024-11-17T09:43:26,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1260ms, sequenceid=441, compaction requested=true 2024-11-17T09:43:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:26,973 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:26,973 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:26,975 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:26,975 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:26,975 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:26,975 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/63fb4f51a9ab4c42a5420fd45e5bcd5f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fe6e7dd017ec4b4088930c0cfc9ba9fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/ca25d32e606a45aca8824a07e3c1d6db] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.4 K 2024-11-17T09:43:26,975 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94282 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:26,975 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:26,976 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:26,976 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/62393bf781af475097c7c933ba9f726b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/8592ddaafb4d42b2ad65ab27a4f2e719, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2381d66321974c68bd8ac3aa0ccf7af9] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=92.1 K 2024-11-17T09:43:26,976 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:26,976 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/62393bf781af475097c7c933ba9f726b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/8592ddaafb4d42b2ad65ab27a4f2e719, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2381d66321974c68bd8ac3aa0ccf7af9] 2024-11-17T09:43:26,976 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62393bf781af475097c7c933ba9f726b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731836600176 2024-11-17T09:43:26,976 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 63fb4f51a9ab4c42a5420fd45e5bcd5f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731836600176 2024-11-17T09:43:26,977 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8592ddaafb4d42b2ad65ab27a4f2e719, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731836602406 2024-11-17T09:43:26,977 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting fe6e7dd017ec4b4088930c0cfc9ba9fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731836602406 2024-11-17T09:43:26,977 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2381d66321974c68bd8ac3aa0ccf7af9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731836603559 2024-11-17T09:43:26,977 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting ca25d32e606a45aca8824a07e3c1d6db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731836603559 2024-11-17T09:43:26,983 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:26,997 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411179d572a4d25fd430da4418688447b8fbb_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:26,998 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:26,999 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/1b7af10f1a8e4fb79dc34adc78f155e7 is 50, key is test_row_0/B:col10/1731836605712/Put/seqid=0 2024-11-17T09:43:27,001 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411179d572a4d25fd430da4418688447b8fbb_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:27,001 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179d572a4d25fd430da4418688447b8fbb_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:27,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742421_1597 (size=12813) 2024-11-17T09:43:27,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742422_1598 (size=4469) 2024-11-17T09:43:27,015 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#501 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:27,017 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/b9785cb5877e42d8b2277fe22ac3af11 is 175, key is test_row_0/A:col10/1731836605712/Put/seqid=0 2024-11-17T09:43:27,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742423_1599 (size=31767) 2024-11-17T09:43:27,027 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/b9785cb5877e42d8b2277fe22ac3af11 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b9785cb5877e42d8b2277fe22ac3af11 2024-11-17T09:43:27,032 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into b9785cb5877e42d8b2277fe22ac3af11(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:27,032 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:27,032 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=13, startTime=1731836606973; duration=0sec 2024-11-17T09:43:27,032 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:27,032 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:27,032 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:27,033 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:27,033 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:27,034 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:27,034 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/206fafee81eb44e48736af1df8ab6eed, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/07453b45c1c94722b934f59161f2f205, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/be2883c152f64d6fb9ea5400d84ec0e6] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.4 K 2024-11-17T09:43:27,034 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 206fafee81eb44e48736af1df8ab6eed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731836600176 2024-11-17T09:43:27,035 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07453b45c1c94722b934f59161f2f205, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731836602406 2024-11-17T09:43:27,035 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting be2883c152f64d6fb9ea5400d84ec0e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731836603559 2024-11-17T09:43:27,047 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:27,047 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/1b6d045f9d34481eba9e1625374a2949 is 50, key is test_row_0/C:col10/1731836605712/Put/seqid=0 2024-11-17T09:43:27,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742424_1600 (size=12813) 2024-11-17T09:43:27,058 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/1b6d045f9d34481eba9e1625374a2949 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1b6d045f9d34481eba9e1625374a2949 2024-11-17T09:43:27,064 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into 1b6d045f9d34481eba9e1625374a2949(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:27,064 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:27,064 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=13, startTime=1731836606973; duration=0sec 2024-11-17T09:43:27,064 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:27,064 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:27,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T09:43:27,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:27,118 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T09:43:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:27,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175cdc4928090345dc9c23bb6d133f06cf_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836605726/Put/seqid=0 2024-11-17T09:43:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T09:43:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742425_1601 (size=12454) 2024-11-17T09:43:27,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:27,143 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175cdc4928090345dc9c23bb6d133f06cf_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175cdc4928090345dc9c23bb6d133f06cf_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:27,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/c3706f4e2bc14e0e83a65d3741b49260, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:27,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/c3706f4e2bc14e0e83a65d3741b49260 is 175, key is test_row_0/A:col10/1731836605726/Put/seqid=0 2024-11-17T09:43:27,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742426_1602 (size=31255) 2024-11-17T09:43:27,415 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/1b7af10f1a8e4fb79dc34adc78f155e7 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/1b7af10f1a8e4fb79dc34adc78f155e7 2024-11-17T09:43:27,419 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into 1b7af10f1a8e4fb79dc34adc78f155e7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:27,419 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:27,419 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=13, startTime=1731836606973; duration=0sec 2024-11-17T09:43:27,420 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:27,420 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T09:43:27,548 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/c3706f4e2bc14e0e83a65d3741b49260 2024-11-17T09:43:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/13a04181c3a54eceadbfec5f0a0bf759 is 50, key is test_row_0/B:col10/1731836605726/Put/seqid=0 2024-11-17T09:43:27,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742427_1603 (size=12301) 2024-11-17T09:43:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:27,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836667910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836667911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,916 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836667914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836667915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836667916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:27,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T09:43:27,959 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/13a04181c3a54eceadbfec5f0a0bf759 2024-11-17T09:43:27,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/172dd390934943938aa702c3f7013f21 is 50, key is test_row_0/C:col10/1731836605726/Put/seqid=0 2024-11-17T09:43:27,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742428_1604 (size=12301) 2024-11-17T09:43:27,970 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/172dd390934943938aa702c3f7013f21 2024-11-17T09:43:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/c3706f4e2bc14e0e83a65d3741b49260 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/c3706f4e2bc14e0e83a65d3741b49260 2024-11-17T09:43:27,976 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/c3706f4e2bc14e0e83a65d3741b49260, entries=150, sequenceid=463, filesize=30.5 K 2024-11-17T09:43:27,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/13a04181c3a54eceadbfec5f0a0bf759 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/13a04181c3a54eceadbfec5f0a0bf759 2024-11-17T09:43:27,980 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/13a04181c3a54eceadbfec5f0a0bf759, entries=150, sequenceid=463, filesize=12.0 K 2024-11-17T09:43:27,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/172dd390934943938aa702c3f7013f21 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/172dd390934943938aa702c3f7013f21 2024-11-17T09:43:27,983 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/172dd390934943938aa702c3f7013f21, entries=150, sequenceid=463, filesize=12.0 K 2024-11-17T09:43:27,983 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0b08a5bb92e2e35ac961ce00a4489f23 in 865ms, sequenceid=463, compaction requested=false 2024-11-17T09:43:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:27,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T09:43:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T09:43:27,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T09:43:27,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1720 sec 2024-11-17T09:43:27,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.1770 sec 2024-11-17T09:43:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:28,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T09:43:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:28,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b3f1f035822b47a8a996cf62697e4c4f_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836608022/Put/seqid=0 2024-11-17T09:43:28,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742429_1605 (size=14994) 2024-11-17T09:43:28,033 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:28,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b3f1f035822b47a8a996cf62697e4c4f_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b3f1f035822b47a8a996cf62697e4c4f_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:28,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/b0e41da09bbb4712ae7b8b0bd9304b24, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:28,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/b0e41da09bbb4712ae7b8b0bd9304b24 is 175, key is test_row_0/A:col10/1731836608022/Put/seqid=0 2024-11-17T09:43:28,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742430_1606 (size=39949) 2024-11-17T09:43:28,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=477, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/b0e41da09bbb4712ae7b8b0bd9304b24 2024-11-17T09:43:28,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/2af7acffa1bf47fbb79069cfb4c76622 is 50, key is test_row_0/B:col10/1731836608022/Put/seqid=0 2024-11-17T09:43:28,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742431_1607 (size=12301) 2024-11-17T09:43:28,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/2af7acffa1bf47fbb79069cfb4c76622 2024-11-17T09:43:28,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836668050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8701ba895d014e9db8c87dd65bf41e5e is 50, key is test_row_0/C:col10/1731836608022/Put/seqid=0 2024-11-17T09:43:28,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742432_1608 (size=12301) 2024-11-17T09:43:28,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8701ba895d014e9db8c87dd65bf41e5e 2024-11-17T09:43:28,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836668054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836668055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836668056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/b0e41da09bbb4712ae7b8b0bd9304b24 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b0e41da09bbb4712ae7b8b0bd9304b24 2024-11-17T09:43:28,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b0e41da09bbb4712ae7b8b0bd9304b24, entries=200, sequenceid=477, filesize=39.0 K 2024-11-17T09:43:28,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/2af7acffa1bf47fbb79069cfb4c76622 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2af7acffa1bf47fbb79069cfb4c76622 2024-11-17T09:43:28,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2af7acffa1bf47fbb79069cfb4c76622, entries=150, sequenceid=477, filesize=12.0 K 2024-11-17T09:43:28,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8701ba895d014e9db8c87dd65bf41e5e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8701ba895d014e9db8c87dd65bf41e5e 2024-11-17T09:43:28,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8701ba895d014e9db8c87dd65bf41e5e, entries=150, sequenceid=477, filesize=12.0 K 2024-11-17T09:43:28,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 0b08a5bb92e2e35ac961ce00a4489f23 in 50ms, sequenceid=477, compaction requested=true 2024-11-17T09:43:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:28,073 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:28,073 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:28,074 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:28,074 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:28,074 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:28,074 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:28,074 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:28,074 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:28,074 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b9785cb5877e42d8b2277fe22ac3af11, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/c3706f4e2bc14e0e83a65d3741b49260, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b0e41da09bbb4712ae7b8b0bd9304b24] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=100.6 K 2024-11-17T09:43:28,074 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/1b7af10f1a8e4fb79dc34adc78f155e7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/13a04181c3a54eceadbfec5f0a0bf759, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2af7acffa1bf47fbb79069cfb4c76622] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.5 K 2024-11-17T09:43:28,074 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:28,074 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b9785cb5877e42d8b2277fe22ac3af11, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/c3706f4e2bc14e0e83a65d3741b49260, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b0e41da09bbb4712ae7b8b0bd9304b24] 2024-11-17T09:43:28,074 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b7af10f1a8e4fb79dc34adc78f155e7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731836603559 2024-11-17T09:43:28,074 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9785cb5877e42d8b2277fe22ac3af11, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731836603559 2024-11-17T09:43:28,075 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a04181c3a54eceadbfec5f0a0bf759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731836605726 2024-11-17T09:43:28,075 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3706f4e2bc14e0e83a65d3741b49260, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731836605726 2024-11-17T09:43:28,075 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 2af7acffa1bf47fbb79069cfb4c76622, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731836607908 2024-11-17T09:43:28,075 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0e41da09bbb4712ae7b8b0bd9304b24, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731836607908 2024-11-17T09:43:28,080 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:28,081 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117b5ae31b061f3444cafacce84cfdada5c_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:28,082 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:28,082 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/97b43af42a1d492ca2f5c3d459984257 is 50, key is test_row_0/B:col10/1731836608022/Put/seqid=0 2024-11-17T09:43:28,083 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117b5ae31b061f3444cafacce84cfdada5c_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:28,083 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b5ae31b061f3444cafacce84cfdada5c_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:28,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742433_1609 (size=12915) 2024-11-17T09:43:28,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742434_1610 (size=4469) 2024-11-17T09:43:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:28,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T09:43:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:28,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173dac4fd71c964f76b5d6b566fcc73e07_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836608045/Put/seqid=0 2024-11-17T09:43:28,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742435_1611 (size=12454) 2024-11-17T09:43:28,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836668176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836668182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836668182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836668183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836668290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836668290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836668290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836668300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,494 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/97b43af42a1d492ca2f5c3d459984257 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/97b43af42a1d492ca2f5c3d459984257 2024-11-17T09:43:28,494 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:28,495 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/84c29bb3d65c45328e99b3117b581397 is 175, key is test_row_0/A:col10/1731836608022/Put/seqid=0 2024-11-17T09:43:28,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742436_1612 (size=31869) 2024-11-17T09:43:28,499 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into 97b43af42a1d492ca2f5c3d459984257(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:28,499 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:28,499 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=13, startTime=1731836608073; duration=0sec 2024-11-17T09:43:28,499 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:28,499 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:28,499 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:28,504 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:28,504 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:28,504 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:28,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,504 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1b6d045f9d34481eba9e1625374a2949, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/172dd390934943938aa702c3f7013f21, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8701ba895d014e9db8c87dd65bf41e5e] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.5 K 2024-11-17T09:43:28,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836668499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836668499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836668499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,505 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b6d045f9d34481eba9e1625374a2949, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731836603559 2024-11-17T09:43:28,506 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 172dd390934943938aa702c3f7013f21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731836605726 2024-11-17T09:43:28,506 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8701ba895d014e9db8c87dd65bf41e5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731836607908 2024-11-17T09:43:28,507 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/84c29bb3d65c45328e99b3117b581397 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/84c29bb3d65c45328e99b3117b581397 2024-11-17T09:43:28,512 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into 84c29bb3d65c45328e99b3117b581397(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:28,512 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:28,513 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=13, startTime=1731836608073; duration=0sec 2024-11-17T09:43:28,513 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:28,513 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:28,513 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:28,514 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/1ae17ae214b64fa5ab920f7381e3230f is 50, key is test_row_0/C:col10/1731836608022/Put/seqid=0 2024-11-17T09:43:28,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836668510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742437_1613 (size=12915) 2024-11-17T09:43:28,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:28,581 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173dac4fd71c964f76b5d6b566fcc73e07_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173dac4fd71c964f76b5d6b566fcc73e07_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:28,582 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d7e2b19adf5241b7b510d814f8b26c42, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:28,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d7e2b19adf5241b7b510d814f8b26c42 is 175, key is test_row_0/A:col10/1731836608045/Put/seqid=0 2024-11-17T09:43:28,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742438_1614 (size=31255) 2024-11-17T09:43:28,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836668810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836668810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836668810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836668819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:28,922 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/1ae17ae214b64fa5ab920f7381e3230f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1ae17ae214b64fa5ab920f7381e3230f 2024-11-17T09:43:28,926 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into 1ae17ae214b64fa5ab920f7381e3230f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:28,926 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:28,926 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=13, startTime=1731836608073; duration=0sec 2024-11-17T09:43:28,926 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:28,926 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T09:43:28,970 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:28,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T09:43:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T09:43:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T09:43:28,972 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T09:43:28,972 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T09:43:28,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T09:43:28,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=498, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d7e2b19adf5241b7b510d814f8b26c42 2024-11-17T09:43:28,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/530984b8df9840f198f706dbc9471325 is 50, key is test_row_0/B:col10/1731836608045/Put/seqid=0 2024-11-17T09:43:28,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742439_1615 (size=12301) 2024-11-17T09:43:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T09:43:29,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T09:43:29,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:29,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T09:43:29,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:29,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T09:43:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836669319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836669319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:29,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836669319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:29,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836669330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:29,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:29,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/530984b8df9840f198f706dbc9471325 2024-11-17T09:43:29,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/e9dc51d8f4924ead9d056d646b38c21f is 50, key is test_row_0/C:col10/1731836608045/Put/seqid=0 2024-11-17T09:43:29,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742440_1616 (size=12301) 2024-11-17T09:43:29,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T09:43:29,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:29,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T09:43:29,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T09:43:29,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T09:43:29,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:29,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:43:29,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/e9dc51d8f4924ead9d056d646b38c21f 2024-11-17T09:43:29,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d7e2b19adf5241b7b510d814f8b26c42 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d7e2b19adf5241b7b510d814f8b26c42 2024-11-17T09:43:29,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d7e2b19adf5241b7b510d814f8b26c42, entries=150, sequenceid=498, filesize=30.5 K 2024-11-17T09:43:29,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/530984b8df9840f198f706dbc9471325 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/530984b8df9840f198f706dbc9471325 2024-11-17T09:43:29,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/530984b8df9840f198f706dbc9471325, entries=150, sequenceid=498, filesize=12.0 K 2024-11-17T09:43:29,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/e9dc51d8f4924ead9d056d646b38c21f as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/e9dc51d8f4924ead9d056d646b38c21f 2024-11-17T09:43:29,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/e9dc51d8f4924ead9d056d646b38c21f, entries=150, sequenceid=498, filesize=12.0 K 2024-11-17T09:43:29,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1661ms, sequenceid=498, compaction requested=false 2024-11-17T09:43:29,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:29,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T09:43:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:29,888 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T09:43:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:29,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:29,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:29,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:29,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:29,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:29,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111731630102a37340d5877dfd58608874c3_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836608181/Put/seqid=0 2024-11-17T09:43:29,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742441_1617 (size=12454) 2024-11-17T09:43:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:29,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. as already flushing 2024-11-17T09:43:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836609986, max timestamp=1731836610017Num uniques 859; 2024-11-17T09:43:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836609986, max timestamp=1731836610017Num uniques 859; 2024-11-17T09:43:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:30,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836609986, max timestamp=1731836610017Num uniques 859; 2024-11-17T09:43:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:30,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836670099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T09:43:30,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836670209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:30,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:30,301 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111731630102a37340d5877dfd58608874c3_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111731630102a37340d5877dfd58608874c3_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:30,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/00b5707d58a34d1199edaa8686f30fee, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:30,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/00b5707d58a34d1199edaa8686f30fee is 175, key is test_row_0/A:col10/1731836608181/Put/seqid=0 2024-11-17T09:43:30,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742442_1618 (size=31255) 2024-11-17T09:43:30,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836670329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836670329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,334 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:30,334 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:30,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836670340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,343 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:30,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836670350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,354 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:30,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836670420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:30,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T09:43:30,706 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=512, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/00b5707d58a34d1199edaa8686f30fee 2024-11-17T09:43:30,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/79edaa72e36c4c5ca9035be31a796164 is 50, key is test_row_0/B:col10/1731836608181/Put/seqid=0 2024-11-17T09:43:30,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742443_1619 (size=12301) 2024-11-17T09:43:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836670730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:30,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:31,118 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/79edaa72e36c4c5ca9035be31a796164 2024-11-17T09:43:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T09:43:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/72f1ed546ddd4597aa690c53b81be43e is 50, key is test_row_0/C:col10/1731836608181/Put/seqid=0 2024-11-17T09:43:31,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742444_1620 (size=12301) 2024-11-17T09:43:31,128 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/72f1ed546ddd4597aa690c53b81be43e 2024-11-17T09:43:31,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/00b5707d58a34d1199edaa8686f30fee as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/00b5707d58a34d1199edaa8686f30fee 2024-11-17T09:43:31,134 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/00b5707d58a34d1199edaa8686f30fee, entries=150, sequenceid=512, filesize=30.5 K 2024-11-17T09:43:31,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/79edaa72e36c4c5ca9035be31a796164 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/79edaa72e36c4c5ca9035be31a796164 2024-11-17T09:43:31,139 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/79edaa72e36c4c5ca9035be31a796164, entries=150, sequenceid=512, filesize=12.0 K 2024-11-17T09:43:31,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/72f1ed546ddd4597aa690c53b81be43e as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/72f1ed546ddd4597aa690c53b81be43e 2024-11-17T09:43:31,142 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/72f1ed546ddd4597aa690c53b81be43e, entries=150, sequenceid=512, filesize=12.0 K 2024-11-17T09:43:31,143 INFO [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1255ms, sequenceid=512, compaction requested=true 2024-11-17T09:43:31,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:31,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:31,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T09:43:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T09:43:31,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T09:43:31,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-11-17T09:43:31,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.1770 sec 2024-11-17T09:43:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:31,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T09:43:31,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:31,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:31,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:31,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:31,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:31,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:31,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b916ba99c8314e33b0ddaf890d1316f1_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836610091/Put/seqid=0 2024-11-17T09:43:31,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742445_1621 (size=14994) 2024-11-17T09:43:31,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836671279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836671390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:31,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836671600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:31,658 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:31,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b916ba99c8314e33b0ddaf890d1316f1_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b916ba99c8314e33b0ddaf890d1316f1_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:31,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/405de43d2ec047b4b01e8a6fdc97af33, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:31,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/405de43d2ec047b4b01e8a6fdc97af33 is 175, key is test_row_0/A:col10/1731836610091/Put/seqid=0 2024-11-17T09:43:31,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742446_1622 (size=39949) 2024-11-17T09:43:31,687 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=549, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/405de43d2ec047b4b01e8a6fdc97af33 2024-11-17T09:43:31,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/74a533a84e944bd3a7e6825add1bd106 is 50, key is test_row_0/B:col10/1731836610091/Put/seqid=0 2024-11-17T09:43:31,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742447_1623 (size=12301) 2024-11-17T09:43:31,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/74a533a84e944bd3a7e6825add1bd106 2024-11-17T09:43:31,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/95e1d6478be040ed9b0d8aee6201d365 is 50, key is test_row_0/C:col10/1731836610091/Put/seqid=0 2024-11-17T09:43:31,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742448_1624 (size=12301) 2024-11-17T09:43:31,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/95e1d6478be040ed9b0d8aee6201d365 2024-11-17T09:43:31,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/405de43d2ec047b4b01e8a6fdc97af33 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/405de43d2ec047b4b01e8a6fdc97af33 2024-11-17T09:43:31,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/405de43d2ec047b4b01e8a6fdc97af33, entries=200, sequenceid=549, filesize=39.0 K 2024-11-17T09:43:31,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/74a533a84e944bd3a7e6825add1bd106 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/74a533a84e944bd3a7e6825add1bd106 2024-11-17T09:43:31,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/74a533a84e944bd3a7e6825add1bd106, entries=150, sequenceid=549, filesize=12.0 K 2024-11-17T09:43:31,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/95e1d6478be040ed9b0d8aee6201d365 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/95e1d6478be040ed9b0d8aee6201d365 2024-11-17T09:43:31,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/95e1d6478be040ed9b0d8aee6201d365, entries=150, sequenceid=549, filesize=12.0 K 2024-11-17T09:43:31,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 0b08a5bb92e2e35ac961ce00a4489f23 in 569ms, sequenceid=549, compaction requested=true 2024-11-17T09:43:31,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:31,812 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:43:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:31,812 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:43:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:31,818 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:43:31,818 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:31,818 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:31,818 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/97b43af42a1d492ca2f5c3d459984257, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/530984b8df9840f198f706dbc9471325, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/79edaa72e36c4c5ca9035be31a796164, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/74a533a84e944bd3a7e6825add1bd106] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=48.7 K 2024-11-17T09:43:31,818 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:43:31,818 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:31,818 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:31,819 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/84c29bb3d65c45328e99b3117b581397, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d7e2b19adf5241b7b510d814f8b26c42, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/00b5707d58a34d1199edaa8686f30fee, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/405de43d2ec047b4b01e8a6fdc97af33] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=131.2 K 2024-11-17T09:43:31,819 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:31,819 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/84c29bb3d65c45328e99b3117b581397, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d7e2b19adf5241b7b510d814f8b26c42, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/00b5707d58a34d1199edaa8686f30fee, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/405de43d2ec047b4b01e8a6fdc97af33] 2024-11-17T09:43:31,821 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 97b43af42a1d492ca2f5c3d459984257, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731836607908 2024-11-17T09:43:31,824 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c29bb3d65c45328e99b3117b581397, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731836607908 2024-11-17T09:43:31,825 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 530984b8df9840f198f706dbc9471325, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731836608045 2024-11-17T09:43:31,825 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7e2b19adf5241b7b510d814f8b26c42, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731836608045 2024-11-17T09:43:31,825 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b5707d58a34d1199edaa8686f30fee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836608175 2024-11-17T09:43:31,826 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 79edaa72e36c4c5ca9035be31a796164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836608175 2024-11-17T09:43:31,826 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 405de43d2ec047b4b01e8a6fdc97af33, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731836610066 2024-11-17T09:43:31,826 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a533a84e944bd3a7e6825add1bd106, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731836610066 2024-11-17T09:43:31,849 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:31,863 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:31,864 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/aa62bb2146e94d479f5da8e8dd70c388 is 50, key is test_row_0/B:col10/1731836610091/Put/seqid=0 2024-11-17T09:43:31,865 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117af6305a44240475ca0d6f42a4be7fb91_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:31,867 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117af6305a44240475ca0d6f42a4be7fb91_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:31,868 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117af6305a44240475ca0d6f42a4be7fb91_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:31,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742449_1625 (size=4469) 2024-11-17T09:43:31,883 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#522 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:31,883 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3e201d5192924eada6783f1c2d4ec9ac is 175, key is test_row_0/A:col10/1731836610091/Put/seqid=0 2024-11-17T09:43:31,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742450_1626 (size=13051) 2024-11-17T09:43:31,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742451_1627 (size=32005) 2024-11-17T09:43:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:31,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T09:43:31,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:31,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:31,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:31,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:31,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:31,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:31,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ff061fc2ee504656ad29388fb01c849c_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836611911/Put/seqid=0 2024-11-17T09:43:31,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742452_1628 (size=14994) 2024-11-17T09:43:32,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836611989, max timestamp=1731836612012Num uniques 859; 2024-11-17T09:43:32,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836611989, max timestamp=1731836612012Num uniques 859; 2024-11-17T09:43:32,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731836611989, max timestamp=1731836612012Num uniques 859; 2024-11-17T09:43:32,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:32,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:32,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/70e37c475432:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T09:43:32,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836672107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836672220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,293 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/aa62bb2146e94d479f5da8e8dd70c388 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/aa62bb2146e94d479f5da8e8dd70c388 2024-11-17T09:43:32,297 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/3e201d5192924eada6783f1c2d4ec9ac as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3e201d5192924eada6783f1c2d4ec9ac 2024-11-17T09:43:32,297 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into aa62bb2146e94d479f5da8e8dd70c388(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:32,297 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:32,297 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=12, startTime=1731836611812; duration=0sec 2024-11-17T09:43:32,297 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:32,298 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:32,298 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T09:43:32,298 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T09:43:32,298 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:32,299 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:32,299 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1ae17ae214b64fa5ab920f7381e3230f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/e9dc51d8f4924ead9d056d646b38c21f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/72f1ed546ddd4597aa690c53b81be43e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/95e1d6478be040ed9b0d8aee6201d365] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=48.7 K 2024-11-17T09:43:32,299 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae17ae214b64fa5ab920f7381e3230f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731836607908 2024-11-17T09:43:32,299 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting e9dc51d8f4924ead9d056d646b38c21f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731836608045 2024-11-17T09:43:32,300 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f1ed546ddd4597aa690c53b81be43e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1731836608175 2024-11-17T09:43:32,300 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e1d6478be040ed9b0d8aee6201d365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731836610066 2024-11-17T09:43:32,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into 3e201d5192924eada6783f1c2d4ec9ac(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:32,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:32,301 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=12, startTime=1731836611811; duration=0sec 2024-11-17T09:43:32,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:32,301 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:32,307 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:32,309 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/a86d1a6e5ea241d18ae8ffcf0191ba74 is 50, key is test_row_0/C:col10/1731836610091/Put/seqid=0 2024-11-17T09:43:32,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742453_1629 (size=13051) 2024-11-17T09:43:32,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:32,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ff061fc2ee504656ad29388fb01c849c_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ff061fc2ee504656ad29388fb01c849c_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:32,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/e36b8b6bc1884d06b8183fb519f6c693, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:32,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/e36b8b6bc1884d06b8183fb519f6c693 is 175, key is test_row_0/A:col10/1731836611911/Put/seqid=0 2024-11-17T09:43:32,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742454_1630 (size=39949) 2024-11-17T09:43:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56150 deadline: 1731836672350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56164 deadline: 1731836672350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56174 deadline: 1731836672350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,355 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,355 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_2' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,356 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_1' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56154 deadline: 1731836672379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,384 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 70e37c475432,40671,1731836432285 for 'test_row_0' in 0b08a5bb92e2e35ac961ce00a4489f23 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-17T09:43:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836672430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,718 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/a86d1a6e5ea241d18ae8ffcf0191ba74 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a86d1a6e5ea241d18ae8ffcf0191ba74 2024-11-17T09:43:32,722 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into a86d1a6e5ea241d18ae8ffcf0191ba74(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:32,722 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:32,722 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=12, startTime=1731836611812; duration=0sec 2024-11-17T09:43:32,722 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:32,722 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:32,731 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=560, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/e36b8b6bc1884d06b8183fb519f6c693 2024-11-17T09:43:32,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/f8f3472b4aec494ea3092f37943963fa is 50, key is test_row_0/B:col10/1731836611911/Put/seqid=0 2024-11-17T09:43:32,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742455_1631 (size=12301) 2024-11-17T09:43:32,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/f8f3472b4aec494ea3092f37943963fa 2024-11-17T09:43:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:43:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40671 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56172 deadline: 1731836672739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 2024-11-17T09:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 , the old value is region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T09:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., hostname=70e37c475432,40671,1731836432285, seqNum=5 because the exception is null or not the one we care about 2024-11-17T09:43:32,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/193648ad3d5246088c91393f17a371cc is 50, key is test_row_0/C:col10/1731836611911/Put/seqid=0 2024-11-17T09:43:32,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742456_1632 (size=12301) 2024-11-17T09:43:33,038 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-17T09:43:33,038 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:33,038 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,038 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:33,039 INFO [Thread-2413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2413. 2024-11-17T09:43:33,039 DEBUG [Thread-2413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:33,039 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,039 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:33,042 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-17T09:43:33,042 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:33,042 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,042 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:33,043 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-17T09:43:33,043 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:33,043 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,043 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:33,044 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-11-17T09:43:33,044 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:33,044 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,044 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T09:43:33,140 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:33,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/193648ad3d5246088c91393f17a371cc 2024-11-17T09:43:33,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/e36b8b6bc1884d06b8183fb519f6c693 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/e36b8b6bc1884d06b8183fb519f6c693 2024-11-17T09:43:33,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/e36b8b6bc1884d06b8183fb519f6c693, entries=200, sequenceid=560, filesize=39.0 K 2024-11-17T09:43:33,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/f8f3472b4aec494ea3092f37943963fa as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/f8f3472b4aec494ea3092f37943963fa 2024-11-17T09:43:33,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/f8f3472b4aec494ea3092f37943963fa, entries=150, sequenceid=560, filesize=12.0 K 2024-11-17T09:43:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/193648ad3d5246088c91393f17a371cc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/193648ad3d5246088c91393f17a371cc 2024-11-17T09:43:33,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/193648ad3d5246088c91393f17a371cc, entries=150, sequenceid=560, filesize=12.0 K 2024-11-17T09:43:33,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1249ms, sequenceid=560, compaction requested=false 2024-11-17T09:43:33,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40671 {}] regionserver.HRegion(8855): Flush requested on 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:33,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T09:43:33,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:33,251 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-11-17T09:43:33,252 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:33,252 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:33,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T09:43:33,252 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:33,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:33,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178289439e20a4471f8aafec4cfc4a4634_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836612084/Put/seqid=0 2024-11-17T09:43:33,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742457_1633 (size=12454) 2024-11-17T09:43:33,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:33,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178289439e20a4471f8aafec4cfc4a4634_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178289439e20a4471f8aafec4cfc4a4634_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:33,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d1a7a436528d4d99badc64f8076f6172, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:33,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d1a7a436528d4d99badc64f8076f6172 is 175, key is test_row_0/A:col10/1731836612084/Put/seqid=0 2024-11-17T09:43:33,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742458_1634 (size=31255) 2024-11-17T09:43:34,067 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=600, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d1a7a436528d4d99badc64f8076f6172 2024-11-17T09:43:34,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/28751191048f44139a18a8be4da0f358 is 50, key is test_row_0/B:col10/1731836612084/Put/seqid=0 2024-11-17T09:43:34,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742459_1635 (size=12301) 2024-11-17T09:43:34,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/28751191048f44139a18a8be4da0f358 2024-11-17T09:43:34,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8eb00c4fcff3408bb73b511fb5ebbd39 is 50, key is test_row_0/C:col10/1731836612084/Put/seqid=0 2024-11-17T09:43:34,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742460_1636 (size=12301) 2024-11-17T09:43:34,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8eb00c4fcff3408bb73b511fb5ebbd39 2024-11-17T09:43:34,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d1a7a436528d4d99badc64f8076f6172 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d1a7a436528d4d99badc64f8076f6172 2024-11-17T09:43:34,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d1a7a436528d4d99badc64f8076f6172, entries=150, sequenceid=600, filesize=30.5 K 2024-11-17T09:43:34,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/28751191048f44139a18a8be4da0f358 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/28751191048f44139a18a8be4da0f358 2024-11-17T09:43:34,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/28751191048f44139a18a8be4da0f358, entries=150, sequenceid=600, filesize=12.0 K 2024-11-17T09:43:34,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/8eb00c4fcff3408bb73b511fb5ebbd39 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8eb00c4fcff3408bb73b511fb5ebbd39 2024-11-17T09:43:34,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8eb00c4fcff3408bb73b511fb5ebbd39, entries=150, sequenceid=600, filesize=12.0 K 2024-11-17T09:43:34,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1645ms, sequenceid=600, compaction requested=true 2024-11-17T09:43:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T09:43:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T09:43:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0b08a5bb92e2e35ac961ce00a4489f23:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T09:43:34,896 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:34,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:34,896 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/B is initiating minor compaction (all files) 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/A is initiating minor compaction (all files) 2024-11-17T09:43:34,897 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/B in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:34,897 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/A in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:34,897 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/aa62bb2146e94d479f5da8e8dd70c388, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/f8f3472b4aec494ea3092f37943963fa, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/28751191048f44139a18a8be4da0f358] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.8 K 2024-11-17T09:43:34,897 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3e201d5192924eada6783f1c2d4ec9ac, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/e36b8b6bc1884d06b8183fb519f6c693, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d1a7a436528d4d99badc64f8076f6172] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=100.8 K 2024-11-17T09:43:34,897 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. files: [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3e201d5192924eada6783f1c2d4ec9ac, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/e36b8b6bc1884d06b8183fb519f6c693, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d1a7a436528d4d99badc64f8076f6172] 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting aa62bb2146e94d479f5da8e8dd70c388, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731836610066 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e201d5192924eada6783f1c2d4ec9ac, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731836610066 2024-11-17T09:43:34,897 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting f8f3472b4aec494ea3092f37943963fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731836611271 2024-11-17T09:43:34,898 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting e36b8b6bc1884d06b8183fb519f6c693, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731836611267 2024-11-17T09:43:34,898 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1a7a436528d4d99badc64f8076f6172, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1731836612084 2024-11-17T09:43:34,898 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 28751191048f44139a18a8be4da0f358, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1731836612084 2024-11-17T09:43:34,903 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:34,904 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/0c74acabfbb5460ab28f1407914eff7a is 50, key is test_row_0/B:col10/1731836612084/Put/seqid=0 2024-11-17T09:43:34,906 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:34,909 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bf909113bead42a1b597c7a5f3cfbda9_0b08a5bb92e2e35ac961ce00a4489f23 store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:34,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742461_1637 (size=13153) 2024-11-17T09:43:34,920 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bf909113bead42a1b597c7a5f3cfbda9_0b08a5bb92e2e35ac961ce00a4489f23, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:34,920 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bf909113bead42a1b597c7a5f3cfbda9_0b08a5bb92e2e35ac961ce00a4489f23 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742462_1638 (size=4469) 2024-11-17T09:43:35,313 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/0c74acabfbb5460ab28f1407914eff7a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/0c74acabfbb5460ab28f1407914eff7a 2024-11-17T09:43:35,316 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/B of 0b08a5bb92e2e35ac961ce00a4489f23 into 0c74acabfbb5460ab28f1407914eff7a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:35,316 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:35,316 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/B, priority=13, startTime=1731836614896; duration=0sec 2024-11-17T09:43:35,316 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T09:43:35,316 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:B 2024-11-17T09:43:35,316 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T09:43:35,317 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T09:43:35,317 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1541): 0b08a5bb92e2e35ac961ce00a4489f23/C is initiating minor compaction (all files) 2024-11-17T09:43:35,317 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0b08a5bb92e2e35ac961ce00a4489f23/C in TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:35,317 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a86d1a6e5ea241d18ae8ffcf0191ba74, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/193648ad3d5246088c91393f17a371cc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8eb00c4fcff3408bb73b511fb5ebbd39] into tmpdir=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp, totalSize=36.8 K 2024-11-17T09:43:35,317 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting a86d1a6e5ea241d18ae8ffcf0191ba74, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731836610066 2024-11-17T09:43:35,317 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 193648ad3d5246088c91393f17a371cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1731836611271 2024-11-17T09:43:35,318 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb00c4fcff3408bb73b511fb5ebbd39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1731836612084 2024-11-17T09:43:35,323 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#C#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:35,323 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/bb165d082ff64d18b15ce90f90112dfd is 50, key is test_row_0/C:col10/1731836612084/Put/seqid=0 2024-11-17T09:43:35,324 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0b08a5bb92e2e35ac961ce00a4489f23#A#compaction#532 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T09:43:35,324 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/1502486fecd349628028f9f9cc1c66ca is 175, key is test_row_0/A:col10/1731836612084/Put/seqid=0 2024-11-17T09:43:35,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742463_1639 (size=13153) 2024-11-17T09:43:35,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742464_1640 (size=32107) 2024-11-17T09:43:35,730 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/bb165d082ff64d18b15ce90f90112dfd as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bb165d082ff64d18b15ce90f90112dfd 2024-11-17T09:43:35,731 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/1502486fecd349628028f9f9cc1c66ca as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/1502486fecd349628028f9f9cc1c66ca 2024-11-17T09:43:35,733 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/C of 0b08a5bb92e2e35ac961ce00a4489f23 into bb165d082ff64d18b15ce90f90112dfd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:35,733 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:35,733 INFO [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/C, priority=13, startTime=1731836614896; duration=0sec 2024-11-17T09:43:35,733 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:35,733 DEBUG [RS:0;70e37c475432:40671-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:C 2024-11-17T09:43:35,735 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0b08a5bb92e2e35ac961ce00a4489f23/A of 0b08a5bb92e2e35ac961ce00a4489f23 into 1502486fecd349628028f9f9cc1c66ca(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T09:43:35,735 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0b08a5bb92e2e35ac961ce00a4489f23: 2024-11-17T09:43:35,735 INFO [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., storeName=0b08a5bb92e2e35ac961ce00a4489f23/A, priority=13, startTime=1731836614896; duration=0sec 2024-11-17T09:43:35,735 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T09:43:35,735 DEBUG [RS:0;70e37c475432:40671-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0b08a5bb92e2e35ac961ce00a4489f23:A 2024-11-17T09:43:36,371 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-11-17T09:43:36,371 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-11-17T09:43:36,371 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:36,371 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:36,371 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,371 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,371 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,371 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:36,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:36,390 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-11-17T09:43:36,390 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:36,390 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,390 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:36,421 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-11-17T09:43:36,421 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T09:43:36,421 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,421 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 164 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2766 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2732 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2793 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2768 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2734 2024-11-17T09:43:36,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T09:43:36,421 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:43:36,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:43:36,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:36,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:36,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T09:43:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T09:43:36,425 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836616424"}]},"ts":"1731836616424"} 2024-11-17T09:43:36,426 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T09:43:36,426 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T09:43:36,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T09:43:36,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, UNASSIGN}] 2024-11-17T09:43:36,428 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, UNASSIGN 2024-11-17T09:43:36,428 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=CLOSING, regionLocation=70e37c475432,40671,1731836432285 2024-11-17T09:43:36,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, UNASSIGN because future has completed 2024-11-17T09:43:36,430 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T09:43:36,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285}] 2024-11-17T09:43:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T09:43:36,581 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:36,581 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T09:43:36,581 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 0b08a5bb92e2e35ac961ce00a4489f23, disabling compactions & flushes 2024-11-17T09:43:36,581 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:36,581 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:36,581 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. after waiting 0 ms 2024-11-17T09:43:36,581 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:36,581 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 0b08a5bb92e2e35ac961ce00a4489f23 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T09:43:36,582 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=A 2024-11-17T09:43:36,582 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:36,582 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=B 2024-11-17T09:43:36,582 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:36,582 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0b08a5bb92e2e35ac961ce00a4489f23, store=C 2024-11-17T09:43:36,582 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T09:43:36,586 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174c63392285404767ba615d523f460903_0b08a5bb92e2e35ac961ce00a4489f23 is 50, key is test_row_0/A:col10/1731836616420/Put/seqid=0 2024-11-17T09:43:36,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742465_1641 (size=12454) 2024-11-17T09:43:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T09:43:36,990 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:43:36,993 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174c63392285404767ba615d523f460903_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174c63392285404767ba615d523f460903_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:36,993 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d67ba43a6f4a42fe885e632a9609ea88, store: [table=TestAcidGuarantees family=A region=0b08a5bb92e2e35ac961ce00a4489f23] 2024-11-17T09:43:36,994 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d67ba43a6f4a42fe885e632a9609ea88 is 175, key is test_row_0/A:col10/1731836616420/Put/seqid=0 2024-11-17T09:43:36,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742466_1642 (size=31255) 2024-11-17T09:43:37,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T09:43:37,398 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=610, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d67ba43a6f4a42fe885e632a9609ea88 2024-11-17T09:43:37,403 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/e17e6dc3b40e42e4b09f18396e1308ac is 50, key is test_row_0/B:col10/1731836616420/Put/seqid=0 2024-11-17T09:43:37,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742467_1643 (size=12301) 2024-11-17T09:43:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T09:43:37,806 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/e17e6dc3b40e42e4b09f18396e1308ac 2024-11-17T09:43:37,811 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/710bc89c2b854c02b424562d53216705 is 50, key is test_row_0/C:col10/1731836616420/Put/seqid=0 2024-11-17T09:43:37,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742468_1644 (size=12301) 2024-11-17T09:43:38,214 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=610 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/710bc89c2b854c02b424562d53216705 2024-11-17T09:43:38,217 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/A/d67ba43a6f4a42fe885e632a9609ea88 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d67ba43a6f4a42fe885e632a9609ea88 2024-11-17T09:43:38,220 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d67ba43a6f4a42fe885e632a9609ea88, entries=150, sequenceid=610, filesize=30.5 K 2024-11-17T09:43:38,220 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/B/e17e6dc3b40e42e4b09f18396e1308ac as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/e17e6dc3b40e42e4b09f18396e1308ac 2024-11-17T09:43:38,223 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/e17e6dc3b40e42e4b09f18396e1308ac, entries=150, sequenceid=610, filesize=12.0 K 2024-11-17T09:43:38,223 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/.tmp/C/710bc89c2b854c02b424562d53216705 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/710bc89c2b854c02b424562d53216705 2024-11-17T09:43:38,226 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/710bc89c2b854c02b424562d53216705, entries=150, sequenceid=610, filesize=12.0 K 2024-11-17T09:43:38,226 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1645ms, sequenceid=610, compaction requested=false 2024-11-17T09:43:38,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/f3bedbbc590d4bd9bf03bb26036ce819, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/0dcda6d95db24f6080c5b9117e210c60, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/87657c3296f2410aa38acfc117327f04, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/92689b9b9d3547568422e9f1f9c263b6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d947328c84cc408cae393fe69efb01b3, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2afebe014a2a4d5e9693ddb8cd33308d, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/553ea4dee8484e62b89b6df9f83adbde, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3902844176384bca8b4814158469466f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3068cd33967b4cc4bd5682c4574a776a, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/02919d9cd0bf4528bd8fd1d0b49b3ff4, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/aa12d45a3cfc43edb272e2320536be59, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/774c9b4a72b74dbfbc33772c094d45c9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/62393bf781af475097c7c933ba9f726b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/8592ddaafb4d42b2ad65ab27a4f2e719, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b9785cb5877e42d8b2277fe22ac3af11, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2381d66321974c68bd8ac3aa0ccf7af9, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/c3706f4e2bc14e0e83a65d3741b49260, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b0e41da09bbb4712ae7b8b0bd9304b24, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/84c29bb3d65c45328e99b3117b581397, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d7e2b19adf5241b7b510d814f8b26c42, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/00b5707d58a34d1199edaa8686f30fee, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/405de43d2ec047b4b01e8a6fdc97af33, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3e201d5192924eada6783f1c2d4ec9ac, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/e36b8b6bc1884d06b8183fb519f6c693, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d1a7a436528d4d99badc64f8076f6172] to archive 2024-11-17T09:43:38,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:43:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/f3bedbbc590d4bd9bf03bb26036ce819 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/f3bedbbc590d4bd9bf03bb26036ce819 2024-11-17T09:43:38,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/0dcda6d95db24f6080c5b9117e210c60 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/0dcda6d95db24f6080c5b9117e210c60 2024-11-17T09:43:38,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/87657c3296f2410aa38acfc117327f04 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/87657c3296f2410aa38acfc117327f04 2024-11-17T09:43:38,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/92689b9b9d3547568422e9f1f9c263b6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/92689b9b9d3547568422e9f1f9c263b6 2024-11-17T09:43:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d947328c84cc408cae393fe69efb01b3 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d947328c84cc408cae393fe69efb01b3 2024-11-17T09:43:38,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2afebe014a2a4d5e9693ddb8cd33308d to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2afebe014a2a4d5e9693ddb8cd33308d 2024-11-17T09:43:38,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/553ea4dee8484e62b89b6df9f83adbde to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/553ea4dee8484e62b89b6df9f83adbde 2024-11-17T09:43:38,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3902844176384bca8b4814158469466f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3902844176384bca8b4814158469466f 2024-11-17T09:43:38,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3068cd33967b4cc4bd5682c4574a776a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3068cd33967b4cc4bd5682c4574a776a 2024-11-17T09:43:38,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/02919d9cd0bf4528bd8fd1d0b49b3ff4 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/02919d9cd0bf4528bd8fd1d0b49b3ff4 2024-11-17T09:43:38,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/aa12d45a3cfc43edb272e2320536be59 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/aa12d45a3cfc43edb272e2320536be59 2024-11-17T09:43:38,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/774c9b4a72b74dbfbc33772c094d45c9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/774c9b4a72b74dbfbc33772c094d45c9 2024-11-17T09:43:38,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/62393bf781af475097c7c933ba9f726b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/62393bf781af475097c7c933ba9f726b 2024-11-17T09:43:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/8592ddaafb4d42b2ad65ab27a4f2e719 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/8592ddaafb4d42b2ad65ab27a4f2e719 2024-11-17T09:43:38,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b9785cb5877e42d8b2277fe22ac3af11 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b9785cb5877e42d8b2277fe22ac3af11 2024-11-17T09:43:38,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2381d66321974c68bd8ac3aa0ccf7af9 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/2381d66321974c68bd8ac3aa0ccf7af9 2024-11-17T09:43:38,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/c3706f4e2bc14e0e83a65d3741b49260 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/c3706f4e2bc14e0e83a65d3741b49260 2024-11-17T09:43:38,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b0e41da09bbb4712ae7b8b0bd9304b24 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/b0e41da09bbb4712ae7b8b0bd9304b24 2024-11-17T09:43:38,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/84c29bb3d65c45328e99b3117b581397 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/84c29bb3d65c45328e99b3117b581397 2024-11-17T09:43:38,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d7e2b19adf5241b7b510d814f8b26c42 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d7e2b19adf5241b7b510d814f8b26c42 2024-11-17T09:43:38,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/00b5707d58a34d1199edaa8686f30fee to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/00b5707d58a34d1199edaa8686f30fee 2024-11-17T09:43:38,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/405de43d2ec047b4b01e8a6fdc97af33 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/405de43d2ec047b4b01e8a6fdc97af33 2024-11-17T09:43:38,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3e201d5192924eada6783f1c2d4ec9ac to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/3e201d5192924eada6783f1c2d4ec9ac 2024-11-17T09:43:38,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/e36b8b6bc1884d06b8183fb519f6c693 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/e36b8b6bc1884d06b8183fb519f6c693 2024-11-17T09:43:38,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d1a7a436528d4d99badc64f8076f6172 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d1a7a436528d4d99badc64f8076f6172 2024-11-17T09:43:38,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/3da9721481a4450e80de587c37b47263, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/b3ee894519f54bba8b5394b3d9f2e341, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fce1a221465c49cca95ba76f1eee7ed6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/bc2a8c69561e4e03a8db242bd58ec258, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2ddf543b9de1472d93451eff1313ef9b, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/762c0fc6947f4541921ee9834e13c614, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/58766491abd64cdabfa2d56754dba062, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c66a79ed066d4288835360f93a2a6853, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/02ab228b4de84afab63ec05aa6c347ad, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/4375e133ca624097a5fdac691dad5b86, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c4f73eb9f0a443e898866e7f8cee7b68, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/63fb4f51a9ab4c42a5420fd45e5bcd5f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c1117078d51d4efe9e739a30c3e396fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fe6e7dd017ec4b4088930c0cfc9ba9fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/1b7af10f1a8e4fb79dc34adc78f155e7, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/ca25d32e606a45aca8824a07e3c1d6db, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/13a04181c3a54eceadbfec5f0a0bf759, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/97b43af42a1d492ca2f5c3d459984257, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2af7acffa1bf47fbb79069cfb4c76622, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/530984b8df9840f198f706dbc9471325, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/79edaa72e36c4c5ca9035be31a796164, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/aa62bb2146e94d479f5da8e8dd70c388, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/74a533a84e944bd3a7e6825add1bd106, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/f8f3472b4aec494ea3092f37943963fa, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/28751191048f44139a18a8be4da0f358] to archive 2024-11-17T09:43:38,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:43:38,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/3da9721481a4450e80de587c37b47263 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/3da9721481a4450e80de587c37b47263 2024-11-17T09:43:38,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/b3ee894519f54bba8b5394b3d9f2e341 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/b3ee894519f54bba8b5394b3d9f2e341 2024-11-17T09:43:38,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fce1a221465c49cca95ba76f1eee7ed6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fce1a221465c49cca95ba76f1eee7ed6 2024-11-17T09:43:38,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/bc2a8c69561e4e03a8db242bd58ec258 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/bc2a8c69561e4e03a8db242bd58ec258 2024-11-17T09:43:38,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2ddf543b9de1472d93451eff1313ef9b to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2ddf543b9de1472d93451eff1313ef9b 2024-11-17T09:43:38,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/762c0fc6947f4541921ee9834e13c614 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/762c0fc6947f4541921ee9834e13c614 2024-11-17T09:43:38,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/58766491abd64cdabfa2d56754dba062 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/58766491abd64cdabfa2d56754dba062 2024-11-17T09:43:38,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c66a79ed066d4288835360f93a2a6853 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c66a79ed066d4288835360f93a2a6853 2024-11-17T09:43:38,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/02ab228b4de84afab63ec05aa6c347ad to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/02ab228b4de84afab63ec05aa6c347ad 2024-11-17T09:43:38,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/4375e133ca624097a5fdac691dad5b86 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/4375e133ca624097a5fdac691dad5b86 2024-11-17T09:43:38,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c4f73eb9f0a443e898866e7f8cee7b68 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c4f73eb9f0a443e898866e7f8cee7b68 2024-11-17T09:43:38,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/63fb4f51a9ab4c42a5420fd45e5bcd5f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/63fb4f51a9ab4c42a5420fd45e5bcd5f 2024-11-17T09:43:38,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c1117078d51d4efe9e739a30c3e396fe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/c1117078d51d4efe9e739a30c3e396fe 2024-11-17T09:43:38,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fe6e7dd017ec4b4088930c0cfc9ba9fe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/fe6e7dd017ec4b4088930c0cfc9ba9fe 2024-11-17T09:43:38,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/1b7af10f1a8e4fb79dc34adc78f155e7 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/1b7af10f1a8e4fb79dc34adc78f155e7 2024-11-17T09:43:38,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/ca25d32e606a45aca8824a07e3c1d6db to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/ca25d32e606a45aca8824a07e3c1d6db 2024-11-17T09:43:38,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/13a04181c3a54eceadbfec5f0a0bf759 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/13a04181c3a54eceadbfec5f0a0bf759 2024-11-17T09:43:38,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/97b43af42a1d492ca2f5c3d459984257 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/97b43af42a1d492ca2f5c3d459984257 2024-11-17T09:43:38,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2af7acffa1bf47fbb79069cfb4c76622 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/2af7acffa1bf47fbb79069cfb4c76622 2024-11-17T09:43:38,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/530984b8df9840f198f706dbc9471325 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/530984b8df9840f198f706dbc9471325 2024-11-17T09:43:38,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/79edaa72e36c4c5ca9035be31a796164 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/79edaa72e36c4c5ca9035be31a796164 2024-11-17T09:43:38,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/aa62bb2146e94d479f5da8e8dd70c388 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/aa62bb2146e94d479f5da8e8dd70c388 2024-11-17T09:43:38,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/74a533a84e944bd3a7e6825add1bd106 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/74a533a84e944bd3a7e6825add1bd106 2024-11-17T09:43:38,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/f8f3472b4aec494ea3092f37943963fa to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/f8f3472b4aec494ea3092f37943963fa 2024-11-17T09:43:38,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/28751191048f44139a18a8be4da0f358 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/28751191048f44139a18a8be4da0f358 2024-11-17T09:43:38,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c98c4ba009034c458d4cbadcea589337, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bc7868ac49d8426da8859551dc24deaf, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/2c0e31dd802f47ada6fb61b99f264679, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/ce99af2cfb55458fbd3d1d74a4c9dc59, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8831286a237c4520949e3e3e3bb18893, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1a87b0d84eff4fa7989f9197afe803fc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/cc29bab920644dd1b7ab54d292cfa3fe, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/3fb23d293c0f4b50897ecafc297bc0ab, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a6792c8825e84fd9a6af8acaab886bb2, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8c9de2c8516349c79b652250bf3a4b79, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8afd3cdfab704c5e838c4e0f4f887d89, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/206fafee81eb44e48736af1df8ab6eed, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c129ff1c677545b4bbbc6ec350242c44, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/07453b45c1c94722b934f59161f2f205, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1b6d045f9d34481eba9e1625374a2949, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/be2883c152f64d6fb9ea5400d84ec0e6, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/172dd390934943938aa702c3f7013f21, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1ae17ae214b64fa5ab920f7381e3230f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8701ba895d014e9db8c87dd65bf41e5e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/e9dc51d8f4924ead9d056d646b38c21f, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/72f1ed546ddd4597aa690c53b81be43e, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a86d1a6e5ea241d18ae8ffcf0191ba74, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/95e1d6478be040ed9b0d8aee6201d365, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/193648ad3d5246088c91393f17a371cc, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8eb00c4fcff3408bb73b511fb5ebbd39] to archive 2024-11-17T09:43:38,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T09:43:38,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c98c4ba009034c458d4cbadcea589337 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c98c4ba009034c458d4cbadcea589337 2024-11-17T09:43:38,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bc7868ac49d8426da8859551dc24deaf to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bc7868ac49d8426da8859551dc24deaf 2024-11-17T09:43:38,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/2c0e31dd802f47ada6fb61b99f264679 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/2c0e31dd802f47ada6fb61b99f264679 2024-11-17T09:43:38,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/ce99af2cfb55458fbd3d1d74a4c9dc59 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/ce99af2cfb55458fbd3d1d74a4c9dc59 2024-11-17T09:43:38,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8831286a237c4520949e3e3e3bb18893 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8831286a237c4520949e3e3e3bb18893 2024-11-17T09:43:38,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1a87b0d84eff4fa7989f9197afe803fc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1a87b0d84eff4fa7989f9197afe803fc 2024-11-17T09:43:38,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/cc29bab920644dd1b7ab54d292cfa3fe to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/cc29bab920644dd1b7ab54d292cfa3fe 2024-11-17T09:43:38,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/3fb23d293c0f4b50897ecafc297bc0ab to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/3fb23d293c0f4b50897ecafc297bc0ab 2024-11-17T09:43:38,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a6792c8825e84fd9a6af8acaab886bb2 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a6792c8825e84fd9a6af8acaab886bb2 2024-11-17T09:43:38,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8c9de2c8516349c79b652250bf3a4b79 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8c9de2c8516349c79b652250bf3a4b79 2024-11-17T09:43:38,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8afd3cdfab704c5e838c4e0f4f887d89 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8afd3cdfab704c5e838c4e0f4f887d89 2024-11-17T09:43:38,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/206fafee81eb44e48736af1df8ab6eed to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/206fafee81eb44e48736af1df8ab6eed 2024-11-17T09:43:38,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c129ff1c677545b4bbbc6ec350242c44 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/c129ff1c677545b4bbbc6ec350242c44 2024-11-17T09:43:38,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/07453b45c1c94722b934f59161f2f205 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/07453b45c1c94722b934f59161f2f205 2024-11-17T09:43:38,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1b6d045f9d34481eba9e1625374a2949 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1b6d045f9d34481eba9e1625374a2949 2024-11-17T09:43:38,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/be2883c152f64d6fb9ea5400d84ec0e6 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/be2883c152f64d6fb9ea5400d84ec0e6 2024-11-17T09:43:38,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/172dd390934943938aa702c3f7013f21 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/172dd390934943938aa702c3f7013f21 2024-11-17T09:43:38,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1ae17ae214b64fa5ab920f7381e3230f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/1ae17ae214b64fa5ab920f7381e3230f 2024-11-17T09:43:38,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8701ba895d014e9db8c87dd65bf41e5e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8701ba895d014e9db8c87dd65bf41e5e 2024-11-17T09:43:38,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/e9dc51d8f4924ead9d056d646b38c21f to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/e9dc51d8f4924ead9d056d646b38c21f 2024-11-17T09:43:38,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/72f1ed546ddd4597aa690c53b81be43e to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/72f1ed546ddd4597aa690c53b81be43e 2024-11-17T09:43:38,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a86d1a6e5ea241d18ae8ffcf0191ba74 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/a86d1a6e5ea241d18ae8ffcf0191ba74 2024-11-17T09:43:38,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/95e1d6478be040ed9b0d8aee6201d365 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/95e1d6478be040ed9b0d8aee6201d365 2024-11-17T09:43:38,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/193648ad3d5246088c91393f17a371cc to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/193648ad3d5246088c91393f17a371cc 2024-11-17T09:43:38,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8eb00c4fcff3408bb73b511fb5ebbd39 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/8eb00c4fcff3408bb73b511fb5ebbd39 2024-11-17T09:43:38,290 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/recovered.edits/613.seqid, newMaxSeqId=613, maxSeqId=4 2024-11-17T09:43:38,290 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. 2024-11-17T09:43:38,290 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 0b08a5bb92e2e35ac961ce00a4489f23: Waiting for close lock at 1731836616581Running coprocessor pre-close hooks at 1731836616581Disabling compacts and flushes for region at 1731836616581Disabling writes for close at 1731836616581Obtaining lock to block concurrent updates at 1731836616581Preparing flush snapshotting stores in 0b08a5bb92e2e35ac961ce00a4489f23 at 1731836616581Finished memstore snapshotting TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731836616582 (+1 ms)Flushing stores of TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23. at 1731836616582Flushing 0b08a5bb92e2e35ac961ce00a4489f23/A: creating writer at 1731836616582Flushing mob file 0b08a5bb92e2e35ac961ce00a4489f23/A: appending metadata at 1731836616586 (+4 ms)Flushing mob file 0b08a5bb92e2e35ac961ce00a4489f23/A: closing flushed file at 1731836616586Flushing 0b08a5bb92e2e35ac961ce00a4489f23/A: appending metadata at 1731836616993 (+407 ms)Flushing 0b08a5bb92e2e35ac961ce00a4489f23/A: closing flushed file at 1731836616994 (+1 ms)Flushing 0b08a5bb92e2e35ac961ce00a4489f23/B: creating writer at 1731836617400 (+406 ms)Flushing 0b08a5bb92e2e35ac961ce00a4489f23/B: appending metadata at 1731836617402 (+2 ms)Flushing 0b08a5bb92e2e35ac961ce00a4489f23/B: closing flushed file at 1731836617402Flushing 0b08a5bb92e2e35ac961ce00a4489f23/C: creating writer at 1731836617809 (+407 ms)Flushing 0b08a5bb92e2e35ac961ce00a4489f23/C: appending metadata at 1731836617811 (+2 ms)Flushing 0b08a5bb92e2e35ac961ce00a4489f23/C: closing flushed file at 1731836617811Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a1fc1: reopening flushed file at 1731836618217 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ccd9e4e: reopening flushed file at 1731836618220 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d9f8995: reopening flushed file at 1731836618223 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 0b08a5bb92e2e35ac961ce00a4489f23 in 1645ms, sequenceid=610, compaction requested=false at 1731836618226 (+3 ms)Writing region close event to WAL at 1731836618288 (+62 ms)Running coprocessor post-close hooks at 1731836618290 (+2 ms)Closed at 1731836618290 2024-11-17T09:43:38,292 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0b08a5bb92e2e35ac961ce00a4489f23, regionState=CLOSED 2024-11-17T09:43:38,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 because future has completed 2024-11-17T09:43:38,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T09:43:38,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 0b08a5bb92e2e35ac961ce00a4489f23, server=70e37c475432,40671,1731836432285 in 1.8640 sec 2024-11-17T09:43:38,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-17T09:43:38,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0b08a5bb92e2e35ac961ce00a4489f23, UNASSIGN in 1.8680 sec 2024-11-17T09:43:38,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T09:43:38,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-17T09:43:38,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836618299"}]},"ts":"1731836618299"} 2024-11-17T09:43:38,300 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T09:43:38,300 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T09:43:38,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8780 sec 2024-11-17T09:43:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T09:43:38,570 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:38,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T09:43:38,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:38,572 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:38,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T09:43:38,573 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:38,574 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,576 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C, FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/recovered.edits] 2024-11-17T09:43:38,578 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/1502486fecd349628028f9f9cc1c66ca to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/1502486fecd349628028f9f9cc1c66ca 2024-11-17T09:43:38,578 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d67ba43a6f4a42fe885e632a9609ea88 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/A/d67ba43a6f4a42fe885e632a9609ea88 2024-11-17T09:43:38,580 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/0c74acabfbb5460ab28f1407914eff7a to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/0c74acabfbb5460ab28f1407914eff7a 2024-11-17T09:43:38,581 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/e17e6dc3b40e42e4b09f18396e1308ac to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/B/e17e6dc3b40e42e4b09f18396e1308ac 2024-11-17T09:43:38,582 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/710bc89c2b854c02b424562d53216705 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/710bc89c2b854c02b424562d53216705 2024-11-17T09:43:38,583 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bb165d082ff64d18b15ce90f90112dfd to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/C/bb165d082ff64d18b15ce90f90112dfd 2024-11-17T09:43:38,585 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/recovered.edits/613.seqid to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23/recovered.edits/613.seqid 2024-11-17T09:43:38,585 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/default/TestAcidGuarantees/0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,585 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T09:43:38,585 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T09:43:38,586 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T09:43:38,588 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709846b4ef1ec4a33acc91293225b2e77_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111709846b4ef1ec4a33acc91293225b2e77_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,589 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dfddfb0505f454abf5c3d4a688da57c_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dfddfb0505f454abf5c3d4a688da57c_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,589 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171a86ed754596407085a7fc3cab67926c_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171a86ed754596407085a7fc3cab67926c_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,590 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172b74bcfa43644e60af185b49c81c4e5d_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172b74bcfa43644e60af185b49c81c4e5d_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,591 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111731630102a37340d5877dfd58608874c3_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111731630102a37340d5877dfd58608874c3_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,592 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173dac4fd71c964f76b5d6b566fcc73e07_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411173dac4fd71c964f76b5d6b566fcc73e07_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,593 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174c63392285404767ba615d523f460903_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174c63392285404767ba615d523f460903_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,593 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757f61c9a942e43be83d86ebdb4427537_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757f61c9a942e43be83d86ebdb4427537_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,594 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175cdc4928090345dc9c23bb6d133f06cf_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175cdc4928090345dc9c23bb6d133f06cf_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,595 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117611009a1dc664161b0402511f91d285e_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117611009a1dc664161b0402511f91d285e_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,596 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117619fe6ebd55443f28d3d17bd07bdedd0_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117619fe6ebd55443f28d3d17bd07bdedd0_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,597 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178289439e20a4471f8aafec4cfc4a4634_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178289439e20a4471f8aafec4cfc4a4634_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,598 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b31ee87495d54f66a535c856d2bca761_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b31ee87495d54f66a535c856d2bca761_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,599 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b3f1f035822b47a8a996cf62697e4c4f_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b3f1f035822b47a8a996cf62697e4c4f_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,599 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b916ba99c8314e33b0ddaf890d1316f1_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b916ba99c8314e33b0ddaf890d1316f1_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,600 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb809246c4464adca527c14abe88e8b2_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb809246c4464adca527c14abe88e8b2_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,601 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e448614cf0fa413692fa914829ddcf1f_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e448614cf0fa413692fa914829ddcf1f_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,602 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f0feab1c1a2e4a05bcb3dce46b4e3787_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f0feab1c1a2e4a05bcb3dce46b4e3787_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ff061fc2ee504656ad29388fb01c849c_0b08a5bb92e2e35ac961ce00a4489f23 to hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ff061fc2ee504656ad29388fb01c849c_0b08a5bb92e2e35ac961ce00a4489f23 2024-11-17T09:43:38,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T09:43:38,605 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:38,607 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T09:43:38,609 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T09:43:38,610 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:38,610 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T09:43:38,610 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731836618610"}]},"ts":"9223372036854775807"} 2024-11-17T09:43:38,611 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T09:43:38,611 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0b08a5bb92e2e35ac961ce00a4489f23, NAME => 'TestAcidGuarantees,,1731836589941.0b08a5bb92e2e35ac961ce00a4489f23.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T09:43:38,611 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T09:43:38,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731836618611"}]},"ts":"9223372036854775807"} 2024-11-17T09:43:38,613 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T09:43:38,613 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T09:43:38,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-17T09:43:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T09:43:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T09:43:38,680 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T09:43:38,690 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=340 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=2212 (was 2323) 2024-11-17T09:43:38,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T09:43:38,690 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:43:38,690 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:43:38,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:38,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:38,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:43:38,690 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T09:43:38,690 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1388394555, stopped=false 2024-11-17T09:43:38,691 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=70e37c475432,44533,1731836431485 2024-11-17T09:43:38,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:43:38,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:43:38,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:43:38,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:43:38,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T09:43:38,693 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:43:38,693 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:43:38,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:38,694 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '70e37c475432,40671,1731836432285' ***** 2024-11-17T09:43:38,694 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T09:43:38,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:43:38,694 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:43:38,694 INFO [RS:0;70e37c475432:40671 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T09:43:38,694 INFO [RS:0;70e37c475432:40671 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T09:43:38,694 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(959): stopping server 70e37c475432,40671,1731836432285 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;70e37c475432:40671. 2024-11-17T09:43:38,695 DEBUG [RS:0;70e37c475432:40671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:43:38,695 DEBUG [RS:0;70e37c475432:40671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T09:43:38,695 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T09:43:38,696 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T09:43:38,696 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T09:43:38,696 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T09:43:38,696 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:38,696 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T09:43:38,696 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T09:43:38,696 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T09:43:38,696 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T09:43:38,697 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T09:43:38,720 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/info/51c4b0eb04f545f098efe83abe56cc40 is 95, key is TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5./info:/1731836462402/DeleteFamily/seqid=0 2024-11-17T09:43:38,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742469_1645 (size=6514) 2024-11-17T09:43:38,792 INFO [regionserver/70e37c475432:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:43:38,897 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:39,097 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:39,123 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/info/51c4b0eb04f545f098efe83abe56cc40 2024-11-17T09:43:39,141 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/ns/d4c960bcd1e447e49767579458029867 is 93, key is TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5./ns:/1731836462324/DeleteFamily/seqid=0 2024-11-17T09:43:39,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742470_1646 (size=5906) 2024-11-17T09:43:39,297 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:39,442 INFO [regionserver/70e37c475432:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T09:43:39,443 INFO [regionserver/70e37c475432:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T09:43:39,497 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:39,545 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/ns/d4c960bcd1e447e49767579458029867 2024-11-17T09:43:39,569 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/rep_barrier/5f170be067db455caecc03bd8bc5f5bc is 102, key is TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5./rep_barrier:/1731836462324/DeleteFamily/seqid=0 2024-11-17T09:43:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742471_1647 (size=6025) 2024-11-17T09:43:39,697 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:39,898 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:39,973 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/rep_barrier/5f170be067db455caecc03bd8bc5f5bc 2024-11-17T09:43:39,990 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/table/dafd7a863a094545b7e93de69f2974df is 96, key is TestAcidGuarantees,,1731836435508.c0fd42d9906a7b067f93ee39238302f5./table:/1731836462324/DeleteFamily/seqid=0 2024-11-17T09:43:39,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742472_1648 (size=5936) 2024-11-17T09:43:40,098 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:40,298 DEBUG [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:43:40,394 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/table/dafd7a863a094545b7e93de69f2974df 2024-11-17T09:43:40,398 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/info/51c4b0eb04f545f098efe83abe56cc40 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/info/51c4b0eb04f545f098efe83abe56cc40 2024-11-17T09:43:40,403 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/info/51c4b0eb04f545f098efe83abe56cc40, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T09:43:40,404 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/ns/d4c960bcd1e447e49767579458029867 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/ns/d4c960bcd1e447e49767579458029867 2024-11-17T09:43:40,407 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/ns/d4c960bcd1e447e49767579458029867, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T09:43:40,408 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/rep_barrier/5f170be067db455caecc03bd8bc5f5bc as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/rep_barrier/5f170be067db455caecc03bd8bc5f5bc 2024-11-17T09:43:40,411 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/rep_barrier/5f170be067db455caecc03bd8bc5f5bc, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T09:43:40,412 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/.tmp/table/dafd7a863a094545b7e93de69f2974df as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/table/dafd7a863a094545b7e93de69f2974df 2024-11-17T09:43:40,415 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/table/dafd7a863a094545b7e93de69f2974df, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T09:43:40,416 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1719ms, sequenceid=90, compaction requested=false 2024-11-17T09:43:40,425 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T09:43:40,426 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T09:43:40,426 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T09:43:40,426 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731836618696Running coprocessor pre-close hooks at 1731836618696Disabling compacts and flushes for region at 1731836618696Disabling writes for close at 1731836618696Obtaining lock to block concurrent updates at 1731836618697 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731836618697Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731836618697Flushing stores of hbase:meta,,1.1588230740 at 1731836618697Flushing 1588230740/info: creating writer at 1731836618698 (+1 ms)Flushing 1588230740/info: appending metadata at 1731836618720 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731836618720Flushing 1588230740/ns: creating writer at 1731836619127 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731836619141 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731836619141Flushing 1588230740/rep_barrier: creating writer at 1731836619548 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731836619569 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731836619569Flushing 1588230740/table: creating writer at 1731836619976 (+407 ms)Flushing 1588230740/table: appending metadata at 1731836619990 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731836619990Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@462ead5b: reopening flushed file at 1731836620398 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@95a292d: reopening flushed file at 1731836620403 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@655ce8a4: reopening flushed file at 1731836620407 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f5a90c5: reopening flushed file at 1731836620412 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1719ms, sequenceid=90, compaction requested=false at 1731836620416 (+4 ms)Writing region close event to WAL at 1731836620422 (+6 ms)Running coprocessor post-close hooks at 1731836620426 (+4 ms)Closed at 1731836620426 2024-11-17T09:43:40,426 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T09:43:40,499 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(976): stopping server 70e37c475432,40671,1731836432285; all regions closed. 2024-11-17T09:43:40,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741834_1010 (size=25344) 2024-11-17T09:43:40,508 DEBUG [RS:0;70e37c475432:40671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/oldWALs 2024-11-17T09:43:40,508 INFO [RS:0;70e37c475432:40671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C40671%2C1731836432285.meta:.meta(num 1731836435106) 2024-11-17T09:43:40,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741833_1009 (size=20353981) 2024-11-17T09:43:40,513 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/WALs/70e37c475432,40671,1731836432285/70e37c475432%2C40671%2C1731836432285.1731836434555 not finished, retry = 0 2024-11-17T09:43:40,616 DEBUG [RS:0;70e37c475432:40671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/oldWALs 2024-11-17T09:43:40,616 INFO [RS:0;70e37c475432:40671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C40671%2C1731836432285:(num 1731836434555) 2024-11-17T09:43:40,616 DEBUG [RS:0;70e37c475432:40671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:43:40,616 INFO [RS:0;70e37c475432:40671 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:43:40,616 INFO [RS:0;70e37c475432:40671 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:43:40,616 INFO [RS:0;70e37c475432:40671 {}] hbase.ChoreService(370): Chore service for: regionserver/70e37c475432:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-17T09:43:40,616 INFO [RS:0;70e37c475432:40671 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:43:40,616 INFO [regionserver/70e37c475432:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:43:40,617 INFO [RS:0;70e37c475432:40671 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40671 2024-11-17T09:43:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:43:40,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/70e37c475432,40671,1731836432285 2024-11-17T09:43:40,625 INFO [RS:0;70e37c475432:40671 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:43:40,625 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [70e37c475432,40671,1731836432285] 2024-11-17T09:43:40,628 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/70e37c475432,40671,1731836432285 already deleted, retry=false 2024-11-17T09:43:40,628 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 70e37c475432,40671,1731836432285 expired; onlineServers=0 2024-11-17T09:43:40,628 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '70e37c475432,44533,1731836431485' ***** 2024-11-17T09:43:40,628 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T09:43:40,628 INFO [M:0;70e37c475432:44533 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:43:40,629 INFO [M:0;70e37c475432:44533 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:43:40,629 DEBUG [M:0;70e37c475432:44533 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T09:43:40,629 DEBUG [M:0;70e37c475432:44533 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T09:43:40,629 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T09:43:40,629 DEBUG [master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836434252 {}] cleaner.HFileCleaner(306): Exit Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836434252,5,FailOnTimeoutGroup] 2024-11-17T09:43:40,629 INFO [M:0;70e37c475432:44533 {}] hbase.ChoreService(370): Chore service for: master/70e37c475432:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T09:43:40,629 INFO [M:0;70e37c475432:44533 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:43:40,629 DEBUG [master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836434251 {}] cleaner.HFileCleaner(306): Exit Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836434251,5,FailOnTimeoutGroup] 2024-11-17T09:43:40,629 DEBUG [M:0;70e37c475432:44533 {}] master.HMaster(1795): Stopping service threads 2024-11-17T09:43:40,629 INFO [M:0;70e37c475432:44533 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T09:43:40,629 INFO [M:0;70e37c475432:44533 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T09:43:40,629 ERROR [M:0;70e37c475432:44533 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T09:43:40,630 INFO [M:0;70e37c475432:44533 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T09:43:40,631 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T09:43:40,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T09:43:40,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:43:40,633 INFO [M:0;70e37c475432:44533 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/.lastflushedseqids 2024-11-17T09:43:40,640 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-17T09:43:40,640 DEBUG [RegionServerTracker-0 {}] master.ActiveMasterManager(353): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-17T09:43:40,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742473_1649 (size=72) 2024-11-17T09:43:40,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:43:40,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40671-0x101254cb3790001, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:43:40,729 INFO [RS:0;70e37c475432:40671 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:43:40,729 INFO [RS:0;70e37c475432:40671 {}] regionserver.HRegionServer(1031): Exiting; stopping=70e37c475432,40671,1731836432285; zookeeper connection closed. 2024-11-17T09:43:40,730 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@250faf5d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@250faf5d 2024-11-17T09:43:40,731 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T09:43:41,046 INFO [M:0;70e37c475432:44533 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T09:43:41,046 INFO [M:0;70e37c475432:44533 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T09:43:41,046 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T09:43:41,046 INFO [M:0;70e37c475432:44533 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:43:41,046 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:43:41,046 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T09:43:41,046 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:43:41,046 INFO [M:0;70e37c475432:44533 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=759.73 KB heapSize=929.59 KB 2024-11-17T09:43:41,061 DEBUG [M:0;70e37c475432:44533 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/12cbcf2f6b9743628f1b5d86f4e4d39a is 82, key is hbase:meta,,1/info:regioninfo/1731836435191/Put/seqid=0 2024-11-17T09:43:41,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742474_1650 (size=5672) 2024-11-17T09:43:41,065 INFO [M:0;70e37c475432:44533 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2095 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/12cbcf2f6b9743628f1b5d86f4e4d39a 2024-11-17T09:43:41,083 DEBUG [M:0;70e37c475432:44533 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d9050b718c994cd09d2ee7a47c6e6ac0 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731836593002/Put/seqid=0 2024-11-17T09:43:41,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742475_1651 (size=44722) 2024-11-17T09:43:41,487 INFO [M:0;70e37c475432:44533 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=759.18 KB at sequenceid=2095 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d9050b718c994cd09d2ee7a47c6e6ac0 2024-11-17T09:43:41,490 INFO [M:0;70e37c475432:44533 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d9050b718c994cd09d2ee7a47c6e6ac0 2024-11-17T09:43:41,505 DEBUG [M:0;70e37c475432:44533 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/422d5298750f4573b2acecf14f2c5293 is 69, key is 70e37c475432,40671,1731836432285/rs:state/1731836434320/Put/seqid=0 2024-11-17T09:43:41,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073742476_1652 (size=5156) 2024-11-17T09:43:41,908 INFO [M:0;70e37c475432:44533 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2095 (bloomFilter=true), to=hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/422d5298750f4573b2acecf14f2c5293 2024-11-17T09:43:41,911 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/12cbcf2f6b9743628f1b5d86f4e4d39a as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/12cbcf2f6b9743628f1b5d86f4e4d39a 2024-11-17T09:43:41,914 INFO [M:0;70e37c475432:44533 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/12cbcf2f6b9743628f1b5d86f4e4d39a, entries=8, sequenceid=2095, filesize=5.5 K 2024-11-17T09:43:41,914 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d9050b718c994cd09d2ee7a47c6e6ac0 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d9050b718c994cd09d2ee7a47c6e6ac0 2024-11-17T09:43:41,916 INFO [M:0;70e37c475432:44533 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d9050b718c994cd09d2ee7a47c6e6ac0 2024-11-17T09:43:41,917 INFO [M:0;70e37c475432:44533 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d9050b718c994cd09d2ee7a47c6e6ac0, entries=166, sequenceid=2095, filesize=43.7 K 2024-11-17T09:43:41,917 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/422d5298750f4573b2acecf14f2c5293 as hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/422d5298750f4573b2acecf14f2c5293 2024-11-17T09:43:41,919 INFO [M:0;70e37c475432:44533 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43475/user/jenkins/test-data/1c8f9bc5-781b-1993-1697-913ef68490be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/422d5298750f4573b2acecf14f2c5293, entries=1, sequenceid=2095, filesize=5.0 K 2024-11-17T09:43:41,920 INFO [M:0;70e37c475432:44533 {}] regionserver.HRegion(3140): Finished flush of dataSize ~759.73 KB/777968, heapSize ~929.30 KB/951600, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 874ms, sequenceid=2095, compaction requested=false 2024-11-17T09:43:41,921 INFO [M:0;70e37c475432:44533 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:43:41,921 DEBUG [M:0;70e37c475432:44533 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731836621046Disabling compacts and flushes for region at 1731836621046Disabling writes for close at 1731836621046Obtaining lock to block concurrent updates at 1731836621046Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731836621046Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=777968, getHeapSize=951840, getOffHeapSize=0, getCellsCount=2286 at 1731836621047 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731836621047Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731836621047Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731836621061 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731836621061Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731836621068 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731836621083 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731836621083Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731836621490 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731836621504 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731836621504Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27eaaf1e: reopening flushed file at 1731836621911 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@525c9ac0: reopening flushed file at 1731836621914 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c952298: reopening flushed file at 1731836621917 (+3 ms)Finished flush of dataSize ~759.73 KB/777968, heapSize ~929.30 KB/951600, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 874ms, sequenceid=2095, compaction requested=false at 1731836621920 (+3 ms)Writing region close event to WAL at 1731836621921 (+1 ms)Closed at 1731836621921 2024-11-17T09:43:41,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38725 is added to blk_1073741830_1006 (size=915326) 2024-11-17T09:43:41,924 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:43:41,924 INFO [M:0;70e37c475432:44533 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T09:43:41,924 INFO [M:0;70e37c475432:44533 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44533 2024-11-17T09:43:41,925 INFO [M:0;70e37c475432:44533 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:43:41,952 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T09:43:41,952 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T09:43:41,953 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T09:43:41,953 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T09:43:42,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:43:42,027 INFO [M:0;70e37c475432:44533 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:43:42,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44533-0x101254cb3790000, quorum=127.0.0.1:60879, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:43:42,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T09:43:42,034 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T09:43:42,034 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T09:43:42,035 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T09:43:42,035 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/hadoop.log.dir/,STOPPED} 2024-11-17T09:43:42,037 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T09:43:42,037 WARN [BP-123921581-172.17.0.2-1731836428229 heartbeating to localhost/127.0.0.1:43475 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T09:43:42,037 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T09:43:42,037 WARN [BP-123921581-172.17.0.2-1731836428229 heartbeating to localhost/127.0.0.1:43475 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-123921581-172.17.0.2-1731836428229 (Datanode Uuid 7f96f725-ead6-4d8e-9771-bfbf62a57c52) service to localhost/127.0.0.1:43475 2024-11-17T09:43:42,039 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5/data/data1/current/BP-123921581-172.17.0.2-1731836428229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T09:43:42,040 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/cluster_2f71c124-7167-12ed-fbcb-be6c1c99bfe5/data/data2/current/BP-123921581-172.17.0.2-1731836428229 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T09:43:42,040 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T09:43:42,050 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T09:43:42,050 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T09:43:42,051 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T09:43:42,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T09:43:42,051 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/676c8511-a7f5-1807-e6ed-2503588336ec/hadoop.log.dir/,STOPPED} 2024-11-17T09:43:42,066 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T09:43:42,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down